wehub/llmware-ai--llmware
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
main
llmware-ai--llmware/llmware/models.py
T
wehub-resource-sync 86db9aae8e
Documentation / build (push) Waiting to run
Details
Documentation / deploy (push) Blocked by required conditions
Details
chore: import upstream snapshot with attribution
2026-07-13 13:34:55 +08:00

16220 lines
602 KiB
Python
Executable File
Raw Permalink Blame History

# Copyright 2023-2026 llmware
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
"""The models module implements the model registry, the catalog for models and prompts, and classes that
implement the interface for each of the supported models. """
import os, logging, json, requests, tempfile, ast, time, shutil, importlib, sys, ctypes
from collections import deque
from importlib import util
from typing import Mapping, Any
from pathlib import Path
from llmware.util import Utilities, AgentWriter, LocalTokenizer
from llmware.configs import (LLMWareConfig, LLMWareException, ModelNotFoundException,
GGUFLibNotLoadedException,DependencyNotInstalledException)
from llmware.model_configs import (global_model_repo_catalog_list, global_model_finetuning_prompt_wrappers_lookup,
global_default_prompt_catalog, model_benchmark_data, global_tokenizer_bos_eos_lookup)
from llmware.gguf_configs import *
from llmware.gguf_configs import _LlamaModel, _LlamaContext, _LlamaBatch, _LlamaTokenDataArray
# torch - import only if needed
# --torch is a required dependency for HFGenerativeModels and HFEmbeddingModels
# --if either of those classes is called, Torch will be imported at that time
torch = None
GLOBAL_TORCH_IMPORT = False
# openvino - import only if needed
# --openvino and openvino_genai are dependencies of OVGenerativeModel
GLOBAL_OVG_IMPORT = False
GLOBAL_OPENVINO_IMPORT = False
ovg = None
openvino = None
ovc = None
# onnxruntime_genai - import only if needed
# -- onnxruntime_genai is dependency of ONNXGenerativeModel
GLOBAL_ONNX_GENAI_RUNTIME = False
og = None
# onnxruntime - import only if needed
# -- onnxruntime is dependency of ONNXEmbeddingModel
# -- it is called implicitly by ONNXGenerativeModel
GLOBAL_ONNX_CORE_RUNTIME = False
ort = None
logger = logging.getLogger(__name__)
logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__))
class _ModelRegistry:
""" ModelRegistry class is wrapper class around the global_model_repo_catalog_list for easy dynamic updating,
and holds most of the key Model, ModelClass and Function/Tool mappings and configurations. """
# notes:
# --held out as internal global cls to keep options to adapt implementation over time
# --generally does not to be directly accessed -> make changes through ModelCatalog
# pulls default model list from model_configs.py
registered_models = global_model_repo_catalog_list
# global list of supported model classes with module lookup - and placeholder for other attributes over time
model_classes = {"ONNXGenerativeModel": {"module": "llmware.models", "open_source": True},
"OVGenerativeModel": {"module": "llmware.models", "open_source": True},
"GGUFGenerativeModel": {"module": "llmware.models", "open_source":True},
"GGUFVisionGenerativeModel": {"module": "llmware.models", "open_source":True},
"OVVisionGenerativeModel": {"module": "llmware.models", "open_source": True},
"ONNXQNNGenerativeModel": {"module": "llmware.models", "open_source":True},
"ONNXEmbeddingModel": {"module": "llmware.models", "open_source": True},
"ONNXVisionGenerativeModel": {"module": "llmware.models", "open_source":True},
"OVEmbeddingModel": {"module": "llmware.models", "open_source": True},
"WindowsLocalFoundryModel": {"module": "llmware.models", "open_source":True},
"WhisperCPPModel": {"module": "llmware.models", "open_source": True},
"HFGenerativeModel": {"module": "llmware.models", "open_source":True},
"HFReRankerModel": {"module": "llmware.models", "open_source": True},
"LLMWareModel": {"module": "llmware.models", "open_source": True},
"LLMWareSemanticModel": {"module": "llmware.models", "open_source": True},
"HFEmbeddingModel": {"module": "llmware.models", "open_source": True},
"OpenChatModel": {"module": "llmware.models", "open_source": True},
"OllamaModel":{"module": "llmware.models", "open_source": True},
"OpenAIGenModel":{"module": "llmware.models", "open_source": False},
"ClaudeModel":{"module": "llmware.models", "open_source": False},
"GoogleGeminiModel":{"module": "llmware.models", "open_source": False},
"OpenAIEmbeddingModel":{"module": "llmware.models", "open_source": False},
}
model_catalog_state_attributes = ["selected_model", "loaded_model_name", "loaded_model_class", "temperature",
"api_endpoint", "get_logits", "max_output", "sample",
"force_reload", "account_name", "library_name", "api_key"]
# model card validation for registering new model - required attributes
min_required_fields = ["model_name", "model_family", "model_category"]
# most fine-tuned models require a specific prompt wrapping that was used in the fine-tuning process
# we are treating these "prompt_wrappers" as core attributes of the model
prompt_wrappers = ["alpaca", "human_bot", "chatgpt", "<INST>", "open_chat", "hf_chat", "chat_ml", "phi_3",
"llama_3_chat","tiny_llama_chat","stablelm_zephyr_chat", "google_gemma_chat",
"vicuna_chat", "phi_4", "deepseek_chat", "phi-4-mini",
"granite_chat", "lfm2_chat", "olmo_chat", "oss_chat", "phi_3_vision"]
registered_wrappers = global_model_finetuning_prompt_wrappers_lookup
# new attribute - track bos/eos for common tokenizers
tokenizer_bos_eos_config = global_tokenizer_bos_eos_lookup
# list of specialized function calling tools
llm_fx_tools = ["ner", "sentiment", "topics", "ratings", "emotions", "nli",
"intent", "sql", "answer", "category", "tags", "summary", "xsum", "extract",
"boolean", "sa-ner","tags-3b", "q_gen", "qa_gen"]
llm_fx_tools_map = {"ner": "slim-ner-tool",
"sentiment": "slim-sentiment-tool",
"topics": "slim-topics-tool",
"ratings": "slim-ratings-tool",
"emotions": "slim-emotions-tool",
"nli": "slim-nli-tool",
"sql": "slim-sql-tool",
"tags": "slim-tags-tool",
"answer": "bling-answer-tool",
"category": "slim-category-tool",
"intent": "slim-intent-tool",
"summary": "slim-summary-tool",
"xsum": "slim-xsum-tool",
"extract": "slim-extract-tool",
"boolean": "slim-boolean-tool",
"sa-ner": "slim-sa-ner-tool",
"tags-3b": "slim-tags-3b-tool",
"q_gen": "slim-q-gen-tiny-tool",
"qa_gen": "slim-qa-gen-tiny-tool"
}
_foundry_manager = None
@classmethod
def get_model_list(cls):
""" List current view of registered models """
return cls.registered_models
@classmethod
def get_model_classes(cls):
""" List of model classes supported in LLMWare. """
return cls.model_classes
@classmethod
def add_model_class(cls, new_class, module="llmware.models", open_source=False,over_write=False):
""" Adds a new model with flexibility to instantiate in new module. By default, it
assumes that the module is the current one, e.g., 'llmware.models'. """
if over_write or new_class not in cls.model_classes:
cls.model_classes.update({new_class:{"module": module, "open_source": open_source}})
elif new_class in cls.model_classes:
logger.warning(f"_ModelRegistry: this model class - {new_class} already exists - to reset the module,"
f"then please pass option over_write=True")
@classmethod
def get_wrapper_list(cls):
""" List current registered wrapper formats """
return cls.registered_wrappers
# new method
@classmethod
def get_tokenizer_bos_eos_lookup(cls):
return cls.tokenizer_bos_eos_config
@classmethod
def get_llm_fx_tools_list (cls):
""" List of function calling model tools available """
return cls.llm_fx_tools
@classmethod
def get_llm_fx_mapping (cls):
""" List of function calling model tools to repo name """
return cls.llm_fx_tools_map
@classmethod
def add_wrapper(cls, wrapper_name, wrapper_dict):
""" Adds a new prompter wrapper to the registered list """
cls.registered_wrappers.update({wrapper_name:wrapper_dict})
cls.prompt_wrappers.append(wrapper_name)
return wrapper_dict
@classmethod
def load_prompt_wrappers_from_file(cls, new_wrapper_registry):
cls.registered_wrappers = {}
cls.prompt_wrappers = []
for key,value in new_wrapper_registry.items():
if key not in cls.prompt_wrappers:
cls.prompt_wrappers.append(key)
cls.registered_wrappers.update({key:value})
@classmethod
def load_tokenizer_configs_from_file(cls, new_tokenizer_configs):
cls.tokenizer_bos_eos_config = {}
for key, value in new_tokenizer_configs.items():
cls.tokenizer_bos_eos_config.update({key:value})
@classmethod
def validate(cls, model_card_dict):
""" Provides minimal validation of structure of a new model card """
for keys in cls.min_required_fields:
if keys not in model_card_dict:
return False
if "model_family" not in model_card_dict:
return False
# removing this condition from validation - provides more extensibility in creating new model classes
"""
if model_card_dict["model_family"] not in cls.model_classes:
return False
"""
if "prompt_wrapper" in model_card_dict:
pwrap = model_card_dict["prompt_wrapper"]
if pwrap:
# ok if prompt_wrapper = ""
if pwrap not in cls.get_wrapper_list():
# permits registering of new model card but issues warning
logger.warning(f"this prompt wrapper - {pwrap} - is not registered which may lead "
f"to unpredictable results in inference - you should register this prompt "
f"format for better results.")
return True
@classmethod
def add_model(cls, model_card_dict, over_write=True):
""" Adds a model to the registry """
if cls.validate(model_card_dict):
# confirm that no overlap in names with model already in the catalog
for i, model in enumerate(cls.registered_models):
if (model["model_name"] in [model_card_dict["model_name"], model_card_dict["display_name"]] or
model["display_name"] in [model_card_dict["model_name"], model_card_dict["display_name"]]):
if not over_write:
raise LLMWareException(message=f"Exception: model name overlaps with another model already "
f"in the ModelCatalog - {model}")
else:
# logger.warning(f"_ModelRegistry - over-write = True - {model['model_name']} - mew model added.")
del cls.registered_models[i]
# go ahead and add model to the catalog
cls.registered_models.append(model_card_dict)
else:
raise LLMWareException(message="New Model Card is Missing Keys")
return model_card_dict
@classmethod
def update_model(cls, model_name_lookup, new_model_card_dict):
""" Updates model in the registry """
if not cls.validate(new_model_card_dict):
raise LLMWareException(message="New Model Card is missing keys.")
updated=False
for i, models in enumerate(cls.registered_models):
# added option to match with display name
if models["model_name"] == model_name_lookup or models["display_name"] == model_name_lookup:
del cls.registered_models[i]
cls.registered_models.append(new_model_card_dict)
updated = True
break
return updated
@classmethod
def delete_model(cls, model_name):
""" Removes model from Model Registry list """
model_found=False
for i, models in enumerate(cls.registered_models):
# added option to match with display name
if models["model_name"] == model_name or models["display_name"] == model_name:
del cls.registered_models[i]
model_found = True
break
if not model_found:
raise ModelNotFoundException(model_name)
return model_found
@classmethod
def new_model_registry(cls, model_registry):
# remove current models
cls.registered_models = []
# add new model registry
for i, model in enumerate(model_registry):
if cls.validate(model):
cls.registered_models.append(model)
return True
@classmethod
def get_model_catalog_vars(cls):
return cls.model_catalog_state_attributes
@classmethod
def add_model_catalog_vars(cls, new_attr):
cls.model_catalog_state_attributes.append(new_attr)
return True
@classmethod
def reset_to_default_catalog(cls):
cls.registered_models = global_model_repo_catalog_list
@classmethod
def get_foundry_manager(cls):
return cls._foundry_manager
@classmethod
def reset_foundry_manager(cls):
cls._foundry_manager = None
return True
@classmethod
def set_foundry_manager(cls, mgr):
cls._foundry_manager = mgr
return mgr
@classmethod
def create_new_foundry_manager(cls):
from foundry_local import FoundryLocalManager
cls._foundry_manager = FoundryLocalManager()
return cls._foundry_manager
def pull_model_from_hf(model_card, local_model_repo_path, api_key=None, **kwargs):
""" Fetches a specific model file from Huggingface repository into local model repo path, generally used for
GGUF models in a repository that contains multiple files - and this method will pull a single designated file.
Inputs: model_card, path to the local model repo, and an api_key (optional). """
from huggingface_hub import hf_hub_download
gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf",
gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
if not os.path.exists(local_model_repo_path):
os.mkdir(local_model_repo_path)
logger.warning(f"Models - pulling model from repo - {gguf_repo} - "
f"and will cache into local folder - {local_model_repo_path}")
try:
downloader = hf_hub_download(gguf_repo, gguf_file, local_dir=local_model_repo_path,
local_dir_use_symlinks=False, token=api_key)
except:
raise LLMWareException(message=f"Models - load_model - pull_model_from_hf - Something has "
f"gone wrong in the download process. Please try again.")
# remove ongoing links, if any, created by attributes not in the file repo
files_created = os.listdir(local_model_repo_path)
if "validation_files" in model_card:
validation_files = model_card["validation_files"]
for files in validation_files:
if files not in files_created:
logger.warning(f"Models - load_model - pull_snapshot_from_hf - missing validation file "
f"expected to run the model correctly - {files}")
if ".huggingface" in files_created:
try:
shutil.rmtree(os.path.join(local_model_repo_path,".huggingface"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .huggingface")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - "
f".huggingface folder created in repo and not auto-removed.")
pass
if ".cache" in files_created:
try:
shutil.rmtree(os.path.join(local_model_repo_path,".cache"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .cache")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - "
f".cache folder created in repo and not auto-removed.")
pass
if ".gitattributes" in files_created:
try:
os.remove(os.path.join(local_model_repo_path, ".gitattributes"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .gitattributes")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - "
f".gitattributes created in repo and not auto-removed.")
pass
return local_model_repo_path
def pull_snapshot_from_hf(model_card, local_model_repo_path, api_key=None, **kwargs):
""" Fetches snapshot of HF model repository and saves into local folder path - two required
inputs:
-- repo_name - the full name of the Huggingface repo, e.g., microsoft/phi-2
-- local_model_repo_path - the local path to save the model files.
"""
from huggingface_hub import snapshot_download
if "gguf_repo" in model_card:
repo_name = model_card["gguf_repo"]
elif "hf_repo" in model_card:
repo_name = model_card["hf_repo"]
elif "ov_repo" in model_card:
repo_name = model_card["ov_repo"]
else:
raise LLMWareException("Model Fetch process error: no repo identified as source to fetch the model.")
# repo_name = model_card["gguf_repo"]
try:
snapshot = snapshot_download(repo_name, local_dir=local_model_repo_path, token=api_key,
local_dir_use_symlinks=False)
except:
raise LLMWareException(message=f"Models - load_model - pull_snapshot_from_hf - {repo_name} - Something has "
f"gone wrong in the download process. Please try again.")
files_created = os.listdir(local_model_repo_path)
logger.debug(f"Models - load_model - pull_snapshot_from_hf - downloaded snapshot - "
f"files cached locally - {files_created}")
if "validation_files" in model_card:
validation_files = model_card["validation_files"]
for files in validation_files:
if files not in files_created:
logger.warning(f"Models - load_model - pull_snapshot_from_hf - missing validation file "
f"expected to run the model correctly - {files}")
# clean up any residual download artifacts in model folder
if ".huggingface" in files_created:
try:
shutil.rmtree(os.path.join(local_model_repo_path,".huggingface"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .huggingface")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - .huggingface folder created in "
f"repo and not auto-removed.")
pass
if ".cache" in files_created:
try:
shutil.rmtree(os.path.join(local_model_repo_path,".cache"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .cache")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - "
f".cache folder created in repo and not auto-removed.")
pass
if ".gitattributes" in files_created:
try:
os.remove(os.path.join(local_model_repo_path, ".gitattributes"))
logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .gitattributes")
except:
logger.info(f"Models - load_model - pull_snapshot_from_hf - .gitattributes created "
f"in repo and not auto-removed.")
pass
return local_model_repo_path
class ModelCatalog:
""" ModelCatalog is the main class responsible for model lookup of (1) Model Card and (2) Finding Model Class.
In most cases, ModelCatalog is the interface for all facets of interacting with the model classes.
"""
def __init__(self):
# ModelCatalog is simple, flexible mechanism to track registered models
# Easy to create "model repo" with mix of model types and instantiation approaches
# Builds on standard model classes with standard inference
self.model_classes = _ModelRegistry().get_model_classes()
self.global_model_list = _ModelRegistry().get_model_list()
self.base_attributes = _ModelRegistry().get_model_catalog_vars()
self.account_name = None
self.library_name= None
# attributes that are used when a model is selected through .load_model method
self.loaded_model_name = None
self.loaded_model_class = None
self.temperature = 0.3
self.use_gpu = True
self.sample = True
self.max_output = 100
self.get_logits = False
self.force_reload = False
self.api_endpoint = None
self.selected_model = None
self.api_key= None
self.custom_loader = None
# new - add - 102024
self.model_kwargs = {}
def to_state_dict(self):
""" Writes selected model state parameters to dictionary. """
state_dict = {}
for keys in self.base_attributes:
if hasattr(self, keys):
state_dict.update({keys: getattr(self, keys)})
return state_dict
def pull_latest_manifest(self):
""" Not implemented currently """
# will add to check manifest in global repo and make available for pull down
return 0
def reset_to_default_catalog(self):
""" Resets model catalog to default list in model_configs """
_ModelRegistry().reset_to_default_catalog()
self.global_model_list = _ModelRegistry().get_model_list()
def save_model_registry(self, fp=None, fn="llmware_model_catalog.json"):
""" Utility method to export global model list to json file """
if not fp:
fp = LLMWareConfig().get_model_repo_path()
json_dict = json.dumps(self.global_model_list, indent=1)
with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile:
outfile.write(json_dict)
return 0
def load_model_registry(self, fp=None, fn="llmware_model_catalog.json"):
""" Utility method to load global model list from json file. Will remove the current
global model list and replace with the model cards from file. """
if not fp:
fp = LLMWareConfig().get_model_repo_path()
model_list = json.load(open(os.path.join(fp,fn), "r"))
_ModelRegistry().new_model_registry(model_list)
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def load_prompt_wrapper_registry(self, fp=None, fn="prompt_wrappers.json"):
""" Utility method to load updated prompt wrapper registry from json file. Will
remove the current global prompt wrapper registry and replace with updated registry from file. """
if not fp:
fp = LLMWareConfig().get_llmware_path()
prompt_list = json.load(open(os.path.join(fp,fn), "r"))
_ModelRegistry().load_prompt_wrappers_from_file(prompt_list)
return True
def save_prompt_wrapper_registry(self, fp=None, fn="prompt_wrappers.json"):
""" Utility method to export global prompt wrapper list to json file """
if not fp:
fp = LLMWareConfig().get_llmware_path()
prompt_list = _ModelRegistry().get_wrapper_list()
json_dict = json.dumps(prompt_list, indent=1)
with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile:
outfile.write(json_dict)
return True
def get_tokenizer_bos_eos_configs(self):
"""" Returns the tokenizer bos eos configs for common models. """
return _ModelRegistry().get_tokenizer_bos_eos_lookup()
def save_tokenizer_bos_eos_configs(self, fp=None, fn="tokenizer_bos_eos_configs.json"):
""" Utility method to export tokenizer bos_eos configs to json file """
if not fp:
fp = LLMWareConfig().get_llmware_path()
tok_configs = _ModelRegistry().get_tokenizer_bos_eos_lookup()
json_dict = json.dumps(tok_configs, indent=1)
with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile:
outfile.write(json_dict)
return True
def load_tokenizer_bos_eos_configs(self, fp=None, fn="tokenizer_bos_eos_configs.json"):
""" Utility method to load updated tokenizer bos_eos configs from json file. Will
remove the current tokenizer bos eos configs and replace with updated configs from file. """
if not fp:
fp = LLMWareConfig().get_llmware_path()
tok_config_list = json.load(open(os.path.join(fp, fn), "r"))
_ModelRegistry().load_tokenizer_configs_from_file(tok_config_list)
return True
def add_model_cards_from_file(self, fp=None, fn="custom_models_manifest.json"):
""" Utility method that loads model cards from a single json file and incrementally adds
to the model global model list. """
if not fp:
fp = LLMWareConfig().get_model_repo_path()
model_add_list = json.load(open(os.path.join(fp, fn), "r"))
for i, model in enumerate(model_add_list):
_ModelRegistry().add_model(model)
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def register_new_model_card(self, model_card_dict):
""" Registers a new model card directly in the model catalog """
_ModelRegistry().add_model(model_card_dict)
# update the global list in ModelCatalog instance
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def delete_model_card(self, model_name):
""" Removes a model card from the registry """
_ModelRegistry().delete_model(model_name)
# update current ModelCatalog instance
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def register_new_finetune_wrapper(self, name, main_start="", main_stop="", llm_start="",
system_start="", system_stop=""):
""" Registers a new fine-tuning wrapper using a basic template that assembles a prompt and will add
special tokens as indicated in the wrapper:
-- main_start - token, if any, to be provided at the start of the prompt template
-- main_stop - token, if any, to be provided at the end of the main 'user' input
-- llm_start - token, if any, at the end of the prompt that is the signal to start the 'assistant' role
-- system_start - optional token to start an initial segment indicating a 'system' instruction
-- system_stop - optional token to stop an initial segment indicating a 'system' instruction.
For example, the LLama-2-Chat wrapper is implemented as follows:
main_start = "<INST>"
main_stop = "</INST>
llm_start = ""
"""
new_dict = {"main_start": main_start, "main_stop": main_stop, "start_llm_response": llm_start,
"system_start": system_start, "system_stop": system_stop}
_ModelRegistry().add_wrapper(name, new_dict)
return 0
def get_list_registered_finetune_wrappers(self):
""" Returns an updated list of registered fine-tuning wrappers. """
return _ModelRegistry().get_wrapper_list()
def register_new_hf_generative_model(self, hf_model_name, llmware_lookup_name=None, display_name=None,
context_window=2048, prompt_wrapper="<INST>",
temperature=0.3, trailing_space="", link=""):
""" Registers any Huggingface Generative Model in the ModelCatalog for easy future lookup and
integration into LLMWare RAG workflows.
The most important input parameter is hf_model_name, which should correspond to the Huggingface Repo/Model
format, e.g., microsoft/phi-2
Any names can be assigned as 'aliases' for the LLMWare Model catalog with both a main lookup name and an
optional secondary lookup to be used as a short-name for screen display.
For example, the 'llmware_lookup_name' for 'microsoft/phi-2' could be 'phi-2'
or 'my-favorite-model-with-2-in-the-name'.
If no llmware_lookup_name is provided, then it will automatically save as the hf_model_name. """
if not llmware_lookup_name:
llmware_lookup_name = hf_model_name
if not display_name:
display_name = hf_model_name
model_card = {"model_name": llmware_lookup_name,
"context_window": context_window,
"prompt_wrapper": prompt_wrapper,
# hf_model_name should correspond to the hf repo/model standard
"hf_repo": hf_model_name,
"display_name": display_name, "temperature": temperature, "trailing_space": trailing_space,
"model_family": "HFGenerativeModel", "model_category": "generative_local",
"model_location": "hf_repo", "instruction_following": False,
"link": link,
"custom_model_files": [], "custom_model_repo": ""}
_ModelRegistry().add_model(model_card)
self.global_model_list = _ModelRegistry().get_model_list()
return model_card
def register_sentence_transformer_model(self, model_name, embedding_dims, context_window,
display_name=None, link=""):
""" Registers a model from the SentenceTransformers library into an LLMWare Model Catalog.
NOTE: for SentenceTransformers, the model_name should match the SentenceTransformer library lookup
name. """
if not display_name:
display_name = model_name
new_model_card_dict = {"model_name": model_name, "context_window": context_window,
"embedding_dims": embedding_dims,
# pre-populated parameters for sentence transformer
"model_family": "LLMWareSemanticModel", "model_category": "embedding",
"display_name": display_name, "link": link,
"model_location": "st_repo",
"custom_model_files": [], "custom_model_repo":""
}
_ModelRegistry().add_model(new_model_card_dict)
self.global_model_list = _ModelRegistry().get_model_list()
return new_model_card_dict
def register_gguf_model(self, model_name, gguf_model_repo, gguf_model_file_name, prompt_wrapper=None,
eos_token_id=0, display_name=None,trailing_space="", temperature=0.3,
context_window=2048, instruction_following=True):
""" Registers a new GGUF model in model catalog - by default, assumes that the GGUF file is in a Huggingface
repository, and will be pulled directly from that repository into a local model_repo cache.
Any arbitrary name can be selected as the model_name and/or display_name for the llmware catalog, as the
core lookup is in the "gguf_repo" and "gguf_file" parameters.
If the GGUF file is in another local file path, then you can access it directly by setting:
"custom_model_repo": "/path/to/local/gguf_model/"
"custom_model_files": "my_model.gguf"
"""
if not display_name:
display_name = model_name
new_model_card_dict = {"model_name": model_name, "display_name": display_name,
"model_family": "GGUFGenerativeModel", "model_category": "generative_local",
"model_location": "llmware_repo", "context_window": context_window,
"instruction_following": instruction_following, "prompt_wrapper": prompt_wrapper,
"temperature": temperature, "trailing_space": trailing_space,
"eos_token_id": eos_token_id,
"gguf_file": gguf_model_file_name,
"gguf_repo": gguf_model_repo,
"link": "", "custom_model_files": [], "custom_model_repo":"",
"fetch": {"module":"llmware.models","method":"pull_model_from_hf"},
"validation_files":[gguf_model_file_name]
}
_ModelRegistry().add_model(new_model_card_dict)
self.global_model_list = _ModelRegistry().get_model_list()
return new_model_card_dict
def register_open_chat_model(self, model_name, api_base=None, model_type="chat", display_name=None,
context_window=4096, instruction_following=True, prompt_wrapper="",
temperature=0.5):
""" Add any open chat model into the LLMWare Model Catalog for easy access, e.g.,
ModelCatalog().register_open_chat_model("my_open_chat_model1", api_base="http://localhost:1234/v1",
prompt_wrapper="<INST>", model_type="chat")
To invoke the model:
my_open_chat_model = ModelCatalog().load_model("my_open_chat_model1")
Or from a prompt:
prompter = Prompt().load_model("my_open_chat_model1")
"""
if not display_name:
display_name = model_name
new_model_card_dict = {"model_name": model_name, "model_type": model_type, "prompt_wrapper": prompt_wrapper,
"display_name": display_name,
"model_family": "OpenChatModel", "model_category": "generative-api",
"model_location": "api", "context_window": context_window,
"instruction_following": instruction_following,
"temperature": temperature, "trailing_space": "",
"api_base": api_base
}
_ModelRegistry().add_model(new_model_card_dict)
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def register_ollama_model(self, model_name, host="localhost", port=11434, model_type="chat",
raw=False, stream=False, display_name=None, context_window=4096,
instruction_following=True, prompt_wrapper="", temperature=0.5):
""" Add any Ollama model into Model Catalog - key parameters:
Assumes -
1. default host/port configs of "localhost:11434"
2. supports 'completion' ollama api, but uses "chat" by default
3. assumes raw=False & stream=False -> more options will be supported over time
If you are using the ollama default settings, then you can register a model card by
simply providing the model name,
e.g., ModelCatalog().register_ollama_model("llama2")
"""
if not display_name:
display_name = model_name
# note: both raw_mode and stream_mode are set to False
new_model_card_dict = {"model_name": model_name, "model_type": model_type,
"host": host, "port": port,
"prompt_wrapper": prompt_wrapper,
"display_name": display_name,
"model_family": "OllamaModel", "model_category": "generative-api",
"model_location": "api", "context_window": context_window,
"instruction_following": instruction_following,
"temperature": temperature, "trailing_space": "",
"raw_mode": False, "stream_mode": False
}
_ModelRegistry().add_model(new_model_card_dict)
self.global_model_list = _ModelRegistry().get_model_list()
return 0
def setup_custom_llmware_inference_server(self, uri_string, secret_key=None):
""" Sets up and registers a custom llmware inference server """
# Examples:
# os.environ["LLMWARE_GPT_URI"] = "http://111.111.1.111:8080"
# os.environ["USER_MANAGED_LLMWARE_GPT_API_KEY"] = "demo-pass-test-key"
# set environ variables with the URL and password key
os.environ["LLMWARE_GPT_URI"] = uri_string
os.environ["USER_MANAGED_LLMWARE_GPT_API_KEY"] = secret_key
return 1
def lookup_model_card (self, selected_model_name):
""" Looks up a model card by model name - the model card has the key configuration and lookup information """
model_card = None
# first check in the global_model_repo + confirm location
for models in self.global_model_list:
# add option to match with display_name as alternative alias for model
if models["model_name"] == selected_model_name or models["display_name"] == selected_model_name:
model_card = models
model_card.update({"standard":True})
break
# if model not found, then return None, and downstream calling function responsible for handling
return model_card
def _instantiate_model_class_from_string(self, model_class, model_name, model_card, api_key=None,
api_endpoint=None, **kwargs):
""" Internal utility method to instantiate model classes from strings. """
# by default - if model not found - return None
my_model = None
context_window= 2048 # used in generative models - use 2048 as default safe backup
embedding_dims = None # used in embedding models
if "context_window" in model_card:
context_window = model_card["context_window"]
if "embedding_dims" in model_card:
embedding_dims = model_card["embedding_dims"]
if model_class in self.model_classes:
module = self.model_classes[model_class]["module"]
model_module = importlib.import_module(module)
if hasattr(model_module, model_class):
model_class = getattr(model_module, model_class)
my_model = model_class(model_name=model_name, context_window=context_window,
api_key=api_key,
trust_remote_code=True,
model_card=model_card,
use_gpu_if_available=self.use_gpu,
get_logits=self.get_logits,
temperature=self.temperature,
max_output=self.max_output,
sample=self.sample,
embedding_dims=embedding_dims,
api_endpoint=api_endpoint,
**kwargs)
else:
raise LLMWareException(message=f"Exception: {model_class} not found.")
return my_model
def model_load_optimizer(self):
""" Enables the ability to intercept the standard model loading process for inserting 'auto optimization'
steps, such as the availability of an API instance of the model or a better performing package, e.g., GGUF
given the intended deployment environment, or even a preferred implementation/version of the model -
without having to change any code.
Currently, not implemented by default, but can be configured to enable custom steps to enable
advanced model routing optimization. """
router_method = ""
router_class = ""
exec_method = None
model_router = LLMWareConfig().get_config("model_router")
router_module = model_router["module"]
if "class" in model_router:
router_class = model_router["class"]
if "method" in model_router:
router_method = model_router["method"]
module = importlib.import_module(router_module)
if router_class:
if hasattr(module, router_class):
exec_class = getattr(module, router_class)()
if hasattr(exec_class, router_method):
exec_method = getattr(exec_class, router_method)
else:
if hasattr(module, router_method):
exec_method = getattr(module, router_method)
if exec_method:
success_dict = exec_method(self.to_state_dict())
if success_dict:
# write attributes, if any, to the ModelCatalog state, which will be picked up
# to "re-direct" the model loading parameters
if isinstance(success_dict, dict):
for k, v in success_dict.items():
setattr(self,k,v)
return True
def load_model (self, selected_model, api_key=None, use_gpu=True, sample=True,get_logits=False,
max_output=100, temperature=-99, force_reload=False, api_endpoint=None,
custom_loader=None, **kwargs):
""" Main method for loading and fully instantiating a model with lookup based on the model_name in
the ModelCatalog. """
# apply optional attributes - will be available to the loaded model
self.use_gpu=use_gpu
self.sample=sample
self.max_output=max_output
self.get_logits=get_logits
self.force_reload = force_reload
self.api_endpoint = api_endpoint
self.selected_model = selected_model
self.api_key=api_key
self.use_gpu = use_gpu
self.custom_loader = custom_loader
# note: temperature set by default at -99, which is a dummy value that is over-ridden by the temperature
# in the model card. This temperature will only be used if explicitly set by the user at value != -99
self.temperature=temperature
# assumed to be set to FALSE in default configs - should not be changed until model route optimizer implemented
if LLMWareConfig().get_config("apply_model_load_router"):
self.model_load_optimizer()
# completes all preparatory steps, and returns 'ready-for-inference' model
selected_model = self.selected_model
logger.debug(f"ModelCatalog - load_model - loading model - {selected_model}")
# step 1- lookup model card from the catalog
model_card = self.lookup_model_card(self.selected_model)
if not model_card:
logger.error(f"error: ModelCatalog - unexpected - could not identify model card for "
f"selected model - {self.selected_model}")
raise ModelNotFoundException(self.selected_model)
# new - 1020 add
if self.model_kwargs:
if not kwargs:
kwargs = {}
for k,v in self.model_kwargs.items():
kwargs.update({k:v})
# end - new add
# step 2- instantiate the right model class
my_model = self.get_model_by_name(model_card["model_name"], api_key=self.api_key,
api_endpoint=self.api_endpoint, **kwargs)
if not my_model:
logger.error(f"error: ModelCatalog - unexpected - could not identify the model - "
f"{self.selected_model}")
raise ModelNotFoundException(self.selected_model)
# step 3- if physical model, then need to locate, validate, potentially fetch and then load
if model_card["model_location"] == "llmware_repo" and not self.api_endpoint:
loading_directions = self.prepare_local_model(model_card,
custom_loader=self.custom_loader,
api_key=self.api_key,
**kwargs)
my_model = my_model.load_model_for_inference(loading_directions, model_card=model_card, **kwargs)
else:
# if api_key passed, save as environ variable
# TODO - look at this
if api_key:
my_model.set_api_key(api_key)
os.environ[selected_model] = api_key
# pass model name to the model directly
my_model.model_name = selected_model
return my_model
def prepare_local_model(self, model_card, custom_loader=None, api_key=None, **kwargs):
""" Resolves obtaining a valid local path to the required model components.
1. Identify if model is available in local path.
-- if custom path provided, then validate from that path.
-- if custom loader provided, then use custom loader to complete this step
-- once local path resolved:
-- Validate that local path contains the required elements
-- Return the loading path to load_the_model_for_inference
2. If not available locally, then need to fetch.
-- Use the fetch method provided in the Model Card
-- if not provided, then use a default for model class
-- need to provide error-handling if download fails
"""
# Step 1 - resolve local path
if custom_loader:
return custom_loader(model_card, api_key=api_key)
if "custom_model_repo" in model_card:
custom_repo = model_card["custom_model_repo"]
else:
custom_repo = None
if custom_repo and os.path.exists(custom_repo):
# if path exists ... (if null result, then will continue down main resolve path)
custom_local_path = self.check_custom_local_repo(model_card, api_key=api_key)
if custom_local_path:
return custom_local_path
# Main resolve path
# check for llmware path & create if not already set up
if not os.path.exists(LLMWareConfig.get_llmware_path()):
# if not explicitly set up by user, then create folder directory structure
LLMWareConfig.setup_llmware_workspace()
if not os.path.exists(LLMWareConfig.get_model_repo_path()):
os.mkdir(LLMWareConfig.get_model_repo_path())
# strip '/' from model name
model_folder_name = model_card["model_name"].split("/")[-1]
model_location = os.path.join(LLMWareConfig.get_model_repo_path(), model_folder_name)
go_ahead = False
if os.path.exists(model_location):
go_ahead = True
model_files = os.listdir(model_location)
if "validation_files" in model_card:
for file in model_card["validation_files"]:
if file not in model_files:
go_ahead = False
break
if len(model_files) == 0:
go_ahead = False
if go_ahead:
return model_location
if not go_ahead:
# need to fetch the model files
fetch, fetch_method_name = self.fetch_resolve(model_card)
if fetch and fetch_method_name:
logger.warning(f"ModelCatalog - load_model - fetching model - {model_card['model_name']} - "
f"from remote repository using {fetch_method_name} - "
f"this may take a couple of minutes the first time.")
# fetch method input: model_card, save_to_path, api_key (optional)
# fetch method must be able to resolve the repo using info in the model card
success = fetch(model_card, model_location, api_key=api_key, **kwargs)
if isinstance(success, dict):
# write attributes, if any, to the Model instance state
for k, v in success.items():
setattr(self, k, v)
return model_location
else:
raise(LLMWareException(message=f"Models - load_model - selected model not found in local path - and "
f"could not identify a supporting fetch method to "
f"retrieve selected model from model repository."))
def fetch_resolve(self, model_card):
""" Returns the fetch method from model card - if not found, then loads default. """
# need to fetch the model -> will use fetch method provided in model card
fetch_module = None
fetch_method = None
fetch_class = None
fetch_exec = None
default_fetch = LLMWareConfig().get_config("model_fetch")
if LLMWareConfig().get_config("apply_default_fetch_override"):
# if set to True, will over-ride the model card and use the default fetch mechanism
fetch_module = default_fetch["module"]
if "class" in default_fetch:
fetch_class = default_fetch["class"]
if "method" in default_fetch:
fetch_method = default_fetch["method"]
else:
# primary (default) case - each model card provides configs for how to fetch the model
if "fetch" in model_card:
if "module" in model_card["fetch"]:
fetch_module = model_card["fetch"]["module"]
if "method" in model_card["fetch"]:
fetch_method = model_card["fetch"]["method"]
if "class" in model_card["fetch"]:
fetch_class = model_card["fetch"]["class"]
if not fetch_module:
# fallback case - if not provided in model card, then fallback to the default fetch mechanism
fetch_module = default_fetch["module"]
if "class" in default_fetch:
fetch_class = default_fetch["class"]
if "method" in default_fetch:
fetch_method = default_fetch["method"]
module = importlib.import_module(fetch_module)
if fetch_class:
if hasattr(module, fetch_class):
class_exec = getattr(module, fetch_class)()
if hasattr(class_exec, fetch_method):
fetch_exec = getattr(class_exec,fetch_method)
else:
if hasattr(module, fetch_method):
fetch_exec = getattr(module, fetch_method)
return fetch_exec, fetch_method
def check_custom_local_repo(self, model_card, api_key=None):
""" Model card provides the option for a custom local path as the execution location for the model.
If 'custom_model_repo' parameter found, then this method will resolve the local path and return
that local path for loading the model. """
# if custom model repo path provided in model card, then pull model from this path
if "custom_model_repo" in model_card:
if model_card["custom_model_repo"]:
if os.path.exists(model_card["custom_model_repo"]):
if "custom_model_files" in model_card:
if model_card["custom_model_files"]:
if len(model_card["custom_model_files"]) > 0:
if os.path.exists(os.path.join(model_card["custom_model_repo"],
model_card["custom_model_files"][0])):
# confirmed that custom path and at least model artifact exist
logger.info(f"update: returning custom model path: "
f"{model_card['custom_model_repo']} - "
f"{model_card['custom_model_files']}")
return model_card["custom_model_repo"]
else:
raise ModelNotFoundException(f"Custom model repo path - {model_card['custom_model_repo']}")
# fallback - if can not validate the path, then will return None and handle in caller
return None
def add_api_key (self, selected_model_name, api_key):
""" Convenience method to apply an api_key to a pass to a model """
# step 1- lookup model card from the catalog
model_card = self.lookup_model_card(selected_model_name)
if not model_card:
logger.error(f"error: ModelCatalog - could not identify model card for "
f"selected model - {selected_model_name}")
raise ModelNotFoundException(selected_model_name)
# step 2 - save api key as environmental variable
model_name = model_card["model_name"]
os.environ[model_name] = api_key
return self
def load_sentence_transformer_model(self,model, model_name):
""" Loads a sentence transformer model """
model = LLMWareSemanticModel(model=model,model_name=model_name)
return model
def load_hf_embedding_model(self, model, tokenizer,trust_remote_code=False):
""" Loads and integrates a Huggingface embedding model """
model = HFEmbeddingModel(model, tokenizer, trust_remote_code=trust_remote_code)
return model
def load_hf_generative_model(self, model,tokenizer,prompt_wrapper=None,
instruction_following=False):
""" Loads and integrates a Huggingface generative decoder-based 'causal' model with limited options
to control model preprocessing prompt behavior """
model = HFGenerativeModel(model, tokenizer, prompt_wrapper=prompt_wrapper,
instruction_following=instruction_following)
return model
def load_embedding_model (self, model_name=None,
model=None, tokenizer=None,from_hf=False,
from_sentence_transformers=False):
""" Loads embedding model by name -
main handler used by any calling function to instantiate embedding model. """
loaded_model = None
# if user passed a 'loaded model' object, then apply directly
if model:
# first, check for 'from_hf' flag and load as HuggingFace model
if from_hf:
loaded_model = ModelCatalog().load_hf_embedding_model(model,tokenizer, trust_remote_code=True)
else:
# second, check for 'from_sentence_transformer' flag and load as SBERT model
if from_sentence_transformers:
loaded_model = ModelCatalog().load_sentence_transformer_model(model,model_name)
if not loaded_model:
logger.error("ModelCatalog - load_embedding_model - could not identify the "
"passed model - if model is from HuggingFace, then mark optional "
"'from_hf' flag to True. If model is from Sentence Transformers, "
"then mark optional 'from_sentence_transformers' flag "
"to True. Note: setting search mode to text search, in absence of embedding "
"model.")
else:
# main case - load embedding model from Catalog
loaded_model = ModelCatalog().load_model(selected_model=model_name)
return loaded_model
def list_open_source_models(self):
""" Lists the open source models in the ModelCatalog. """
open_source_models = []
open_source_class = []
model_classes = _ModelRegistry().get_model_classes()
for key, value in model_classes.items():
if "open_source" in value:
if value["open_source"]:
open_source_class.append(key)
for x in self.global_model_list:
if x["model_family"] in open_source_class:
open_source_models.append(x)
return open_source_models
def list_models_by_type(self, model_family):
model_list = []
# e.g., model_family = "WindowsLocalFoundryModel"
for model in self.global_model_list:
if model["model_family"].lower() == model_family.lower():
model_list.append(model)
return model_list
def list_embedding_models(self):
""" Lists the embedding models in the ModelCatalog. """
embedding_models = []
for x in self.global_model_list:
if x["model_category"] == "embedding":
embedding_models.append(x)
return embedding_models
def list_generative_models(self):
""" Lists the generative models in the ModelCatalog. """
gen_models = []
for x in self.global_model_list:
if x["model_category"].startswith("generative"):
gen_models.append(x)
gen_models = sorted(gen_models, key=lambda x: x["model_name"], reverse=False)
return gen_models
def list_generative_local_models(self):
""" Lists the generative local models in the ModelCatalog. """
gen_local_models = []
for x in self.global_model_list:
if x["model_category"] == "generative_local":
gen_local_models.append(x)
gen_local_models = sorted(gen_local_models, key=lambda x:x["model_name"], reverse=False)
return gen_local_models
def list_all_models(self):
""" Lists all models in the ModelCatalog. """
all_models = []
for x in self.global_model_list:
all_models.append(x)
all_models = sorted(all_models, key=lambda x: x["model_category"], reverse=False)
return all_models
def list_intel_npu_optimized_models(self):
npu_models = []
for model_card in self.global_model_list:
npu_optimized = model_card.get("npu_optimized","")
if npu_optimized:
npu_models.append(model_card)
return npu_models
def model_lookup(self,model_name):
""" Looks up model by model_name. Will check both the primary 'model_name' and the secondary/optional
display_name to look for a match in the ModelCatalog. """
my_model = None
for models in self.global_model_list:
# add check for match with display_name as alias
if models["model_name"] == model_name or models["display_name"] == model_name:
my_model = models
break
return my_model
def get_model_by_name(self, model_name, api_key=None, api_endpoint=None, **kwargs):
""" Gets and instantiates model by name. """
my_model = None
for models in self.global_model_list:
# add check for display name match
if models["model_name"] == model_name or models["display_name"] == model_name:
selected_model = models
my_model = self._instantiate_model_class_from_string(selected_model["model_family"],
model_name, models,api_key=api_key,
api_endpoint=api_endpoint, **kwargs)
break
return my_model
def save_benchmark_report(self, fp=None,fn=None):
""" Saves model benchmark score data to jsonl file. Optional inputs to assign folder path (fp) and
filename (fn). If not provided, then will be saved in llmware_data path with default name.
"""
if not fp:
fp = LLMWareConfig().get_llmware_path()
if not fn:
fn = "llmware_model_benchmark_scores"
test_fn = fn + ".jsonl"
f_out = open(os.path.join(fp, test_fn), "w")
for entry in model_benchmark_data:
jsonl_row = json.dumps(entry)
f_out.write(jsonl_row)
f_out.write("\n")
f_out.close()
return fp
def get_benchmark_score(self, model_name):
""" Looks up benchmark score for a model, if available. Returns None if no benchmark available. """
for i, entry in enumerate(model_benchmark_data):
if entry["model_name"] == model_name:
return entry
logger.debug(f"ModelCatalog - get_benchmark_score - {model_name} does not have a benchmark available.")
return None
def get_benchmark_by_filter (self, conditions=None):
""" Will apply a list of {key:value} conditions to provide a subset of models that fit the conditions.
Conditions are a list of dictionaries, with each dictionary entry consisting of the following:
-- {key, "eval str"},
-- e.g., {"parameters", "parameters < 3"}
To create multiple conditions - create a list of several dictionaries:
-- e.g., [ {"parameters", "parameters < 6"}, {"accuracy_score", "accuracy_score > 95"} ]
"""
if not conditions:
logger.debug("ModelCatalog - get_benchmark_by_filter - no conditions provided, so returning all of the "
"benchmark data list.")
return model_benchmark_data
if isinstance(conditions,dict):
conditions = [conditions]
else:
if not isinstance(conditions,list):
logger.warning(f"ModelCatalog - conditions should be structured as a list of dictionary entries, "
f"with each dictionary entry consisting of a pair of a key:eval_str")
return model_benchmark_data
results = []
for i, entry in enumerate(model_benchmark_data):
num_conditions = 0
true_conditions = 0
for cond in conditions:
if isinstance(cond, dict):
num_conditions += 1
for key,value in cond.items():
if key in entry:
truth_value = eval(value, {key:entry[key]})
if truth_value:
true_conditions += 1
if num_conditions > 0 and num_conditions == true_conditions:
results.append(entry)
return results
def get_llm_toolkit(self, tool_list=None, api_key=None):
""" Caches all SLIM tools by default, or if list provided, then selected tools only. """
model_repo_path = LLMWareConfig.get_model_repo_path()
if not os.path.exists(model_repo_path):
os.makedirs(model_repo_path)
if not tool_list:
tool_list = _ModelRegistry().get_llm_fx_tools_list()
for tool in tool_list:
tool_name = _ModelRegistry().get_llm_fx_mapping()[tool]
logger.info(f"ModelCatalog - get_toolset - {tool} - {tool_name}")
found_model = False
local_model_repo_path = os.path.join(model_repo_path, tool_name)
if os.path.exists(local_model_repo_path):
model_parts_in_folder = os.listdir(local_model_repo_path)
if len(model_parts_in_folder) > 0:
found_model = True
if not found_model:
model_card = self.lookup_model_card(tool_name)
pull_snapshot_from_hf(model_card, local_model_repo_path, api_key=api_key)
return 0
def list_llm_tools(self):
"""Provides a list of the currently available SLIM tools available in the catalog. """
return _ModelRegistry().get_llm_fx_tools_list()
def get_llm_fx_mapping(self):
"""Provides a current mapping of Tools to LLM Function Call - this mapping is used by LLMfx class to
orchestrate among multiple models deployed locally as tools. """
return _ModelRegistry().get_llm_fx_mapping()
def get_test_script(self, model_name):
""" Checks if a test script is available with the model repo - and if so,
retrieves the test set as a json dictionary """
test_set = None
model_repo_path = LLMWareConfig().get_model_repo_path()
local_model_path = os.path.join(model_repo_path, model_name)
if os.path.exists(local_model_path):
model_files = os.listdir(local_model_path)
if "config.json" in model_files:
config_json = json.load(open(os.path.join(local_model_path, "config.json"), "r",
encoding="utf-8"))
if "test_set" in config_json:
test_set = config_json["test_set"]
return test_set
def tool_test_run(self, model_name, api_key=None, verbose=False,
# add more optional configurations to flow thru to the model inference
use_gpu=True, sample=True, get_logits=True,
max_output=100, temperature=-99, custom_test_script=None,
api_endpoint=None):
""" Loads a tool, if required, and executes a series of test runs. Most of the input
parameters are optional configuration parameters that will be passed when the model is loaded
and instantiated.
Note: only available for GGUF quantized 'tool' implementation models. """
model_card = self.lookup_model_card(model_name)
agent_writer = AgentWriter()
if not model_card:
raise ModelNotFoundException(model_name)
model = self.load_model(model_name, api_key=api_key, use_gpu=use_gpu, sample=sample,
get_logits=get_logits,max_output=max_output, temperature=temperature,
api_endpoint=api_endpoint)
if custom_test_script:
# custom_test_script can be any json file with list of json dictionary entries with
# keys corresponding to test set, e.g., "context", "query", "answer"
test_set = custom_test_script
else:
test_set = self.get_test_script(model_name)
if test_set:
if "function_call" not in model_card:
# run traditional inference on test set
agent_writer.write(f"\nTest: {model_name}")
for i, entries in enumerate(test_set):
agent_writer.write(f"\nupdate: query - {i} - {entries['query']}")
response = model.inference(entries["query"],add_context=entries["context"],
add_prompt_engineering="default_with_context")
agent_writer.write(f"\nupdate: llm_response - {i} - {response['llm_response']}")
if "answer" in entries:
agent_writer.write(f"update: gold answer - {i} - {entries['answer']}")
else:
agent_writer.write(f"\nTest: {model_name}")
for i, entries in enumerate(test_set):
text = entries["context"]
# special case for nli
if "conclusion" in entries:
text = "Evidence: " + text + "\nConclusion: " + entries["conclusion"]
# special case for boolean (question = params)
if "question" in entries:
params = entries["question"] + " (explain)"
response = model.function_call(text, params=[params])
else:
# general case - use default params and function from model card
response = model.function_call(text)
# if verbose:
agent_writer.write(f"\nupdate: context - test - {i} - {text}")
agent_writer.write(f"update: 'llm_response' - test - {i} - {response['llm_response']}")
logit_analysis = self.logit_analysis(response, model_card, model.hf_tokenizer_name,
api_key=api_key)
if "ryg_string" in logit_analysis:
agent_writer.write(f"update: red-yellow-green confidence - {logit_analysis['ryg_string']}")
if "confidence_score" in logit_analysis:
agent_writer.write(f"update: confidence score - {logit_analysis['confidence_score']}")
if "marker_tokens" in logit_analysis:
if logit_analysis["marker_tokens"]:
agent_writer.write(f"update: marker tokens - {logit_analysis['marker_tokens']}")
if "choices" in logit_analysis:
choices = logit_analysis["choices"]
if len(choices) > 0:
choices = choices[0]
agent_writer.write(f"update: choices - {choices}")
agent_writer.close()
return 0
def list_function_call_models(self):
""" Returns a list of model card dictionaries for models that implement function_calls."""
fc_model_list = []
for models in self.global_model_list:
if "function_call" in models:
# confirm that value is positive
if models["function_call"]:
fc_model_list.append(models)
return fc_model_list
def logit_analysis(self, response, model_card, hf_tokenizer_name,api_key=None):
""" Analyzes logits from llm response - currently exposed only as option for function
call inferences in HFGenerative and GGUFGenerative models. """
logit_analysis = []
ryg_string = ""
vz_choices = []
marker_token_probs = []
low_confidence_choices = []
confidence_score = -1
# only go ahead if logits found in response
if "logits" not in response:
logger.warning("ModelCatalog - logit_analysis requires a response dictionary with 'logits' key- skipping")
return logit_analysis
try:
from colorama import Fore
red = Fore.RED
green = Fore.GREEN
yellow = Fore.YELLOW
color_reset = Fore.RESET
except:
logger.warning("ModelCatalog - logit analysis - could not import colorama - please import to see color coded"
"visualization of the output string confidence level.")
# setting color inserts to empty
red = ""
green = ""
yellow = ""
color_reset = ""
""" Analyzes logits from llm response """
# marker tokens for sentiment analysis
marker_tokens = []
marker_token_lookup = {}
if "marker_tokens" in model_card:
marker_tokens = model_card["marker_tokens"]
if "marker_token_lookup" in model_card:
marker_token_lookup = model_card["marker_token_lookup"]
if "logits" in response:
logits = response["logits"]
# tokenizer load
if "tokenizer_local" in model_card:
tokenizer = LocalTokenizer(tokenizer_fn=model_card["tokenizer_local"])
elif util.find_spec("transformers"):
# hf tokenizer name
pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=True, custom_loader=None)
tokenizer = pt_loader.get_tokenizer(hf_tokenizer_name)
else:
raise LLMWareException(message="Exception: could not identify tokenizer to use")
try:
# pull bos attributes from tokenizer
# -- note: will be a list of .bos_id and .eos_id, e.g., [2], not 2
bos_token_id = tokenizer.bos_id
bos_str = tokenizer.bos_token
eos_token_id = tokenizer.eos_id
eos_str = tokenizer.eos_token
if not isinstance(eos_token_id, list):
eos_token_id = [eos_token_id]
if isinstance(bos_token_id, list):
if len(bos_token_id) > 0:
bos_token_id = bos_token_id[0]
else:
# set to llama as fallback
bos_token_id = 1
except:
# unexpected - but if fail, then take llama defaults
bos_token_id = 1
bos_str = "<s>"
eos_token_id = [2]
eos_str = "</s>"
ryg_string = ""
token_probs = []
marker_token_probs = []
vz_choices = []
vz_capture_on = False
for i, toks in enumerate(response["output_tokens"]):
# change - look directly for '[' in tokenized output
if "]" in tokenizer.decode(toks):
vz_capture_on = False
if toks in marker_tokens:
for x in range(0, len(logits[i])):
if logits[i][x][0] in marker_tokens:
# new add 1020 - if from file, then dict number converted to str
if logits[i][x][0] in marker_token_lookup:
entry0 = marker_token_lookup[logits[i][x][0]]
elif str(logits[i][x][0]) in marker_token_lookup:
entry0 = marker_token_lookup[str(logits[i][x][0])]
else:
entry0 = "NA"
# end here
new_entry = (entry0,
logits[i][x][0],
logits[i][x][1])
marker_token_probs.append(new_entry)
if vz_capture_on:
new_entry = {}
for x in range(0,3):
key = "choice_" + str(x+1)
new_entry.update({key: [tokenizer.decode(logits[i][x][0]),
logits[i][x][1],logits[i][x][0]]})
# set confidence score as normalized logit value of first token in value zone
#TODO: need to assess whether averaging across multiple tokens more effective
if len(vz_choices) == 0:
if logits[i][x][0] == toks:
confidence_score = logits[i][x][1]
vz_choices.append(new_entry)
# change - look for "[" directly in token decoded output
if "[" in tokenizer.decode(toks):
vz_capture_on = True
# e.g., if toks in [2]:
if toks in eos_token_id:
break
for x in range(0, len(logits[i])):
if toks == logits[i][x][0]:
token_probs.append(logits[i][x][1])
if logits[i][x][1] > 0.70:
ryg_string += green + tokenizer.decode([bos_token_id, logits[i][x][0]])
if 0.3 <= logits[i][x][1] <= 0.70:
ryg_string += yellow + tokenizer.decode([bos_token_id, logits[i][x][0]])
new_entry = {}
for y in range(0, 3):
key = "choice_" + str(y + 1)
new_entry.update({key: [tokenizer.decode(logits[i][y][0]),
logits[i][y][1], logits[i][y][0]]})
low_confidence_choices.append(new_entry)
if logits[i][x][1] < 0.3:
ryg_string += red + tokenizer.decode([bos_token_id, logits[i][x][0]])
new_entry = {}
for y in range(0, 3):
key = "choice_" + str(y + 1)
new_entry.update({key: [tokenizer.decode(logits[i][y][0]),
logits[i][y][1], logits[i][y][0]]})
low_confidence_choices.append(new_entry)
# removing hard-coded "<s>"
ryg_string = ryg_string.replace(bos_str, "")
logit_analysis = {"ryg_string": ryg_string + color_reset, "choices": vz_choices,
"marker_tokens": marker_token_probs,
"low_confidence_choices": low_confidence_choices,
"confidence_score": confidence_score}
return logit_analysis
def fc_output_values(self, model_name):
""" Takes as input a model_name, and if the model is function-calling, then will output a list
of the expected function calling output values for the model. If no value provided, or no specific
expected 'constraints' on output values, then returns an empty list. """
output_values = []
model_card = self.lookup_model_card(model_name)
if model_card:
if "fc_output_values" in model_card:
output_values = model_card["fc_output_values"]
else:
logger.error(f"ModelCatalog - could not identify model card "
f"for selected model - {model_name} ")
raise ModelNotFoundException(model_name)
return output_values
def fc_primary_keys(self, model_name):
""" Takes as input a model_name, and if the model is function-calling, then will output a list of the
primary keys, if any, to be passed as parameters to the model. If no primary keys, then returns an
empty list. """
output_keys = []
model_card = self.lookup_model_card(model_name)
if model_card:
if "primary_keys" in model_card:
output_keys = model_card["primary_keys"]
else:
logger.error(f"ModelCatalog - could not identify model card for "
f"selected model - {model_name}")
raise ModelNotFoundException(model_name)
return output_keys
def remediate_function_call_string(self,input_string, dedupe_values=True):
""" This method attempts to remediate a function call output string that can not be automatically
converted into a programmatic object. The method supports both DICT and LIST outputs. It is designed
to address the most common source of automatic failing, which is a premature termination at the end of the
string, usually due to a max_len cap, e.g., {'key': ['value1', value2', ..., 'val """
starter = 3
keys = []
values = []
# if very short output, then can not remediate - assume that a bigger problem happened with the inference
if len(input_string) < starter:
# llm response very short - could not remediate and convert to dict or list
return "string", input_string
start = -1
list_start = -1
# will scan the start of the string for either a dictionary start '{' or list start '['
# if neither found, will return the original string
for x in range(0, starter):
if input_string[x] == "{":
# found dict starter
start = x
if input_string[x] == "[":
# found list starter
list_start = x
if start < 0 and list_start < 0:
# remediation not successful - could not find a start marker for dictionary or list
return "string", input_string
# based on the start marker, determine the target output type
if start < 0 and list_start >= 0:
# try to build the string as a list output
list_type = True
key_or_value = "value"
response_type = "list"
start = list_start-1
else:
# try to build the string as a dictionary output
list_type = False
key_or_value = "key"
response_type = "dict"
string_on = False
key_tmp = ""
counter = 0
output_dict = {}
output_list = []
current_key = ""
logger.debug(f"***test*** - remediation - input string - {input_string}")
for y in range(start + 1, len(input_string)):
# note: ASCII ORD conversion - 58 - ':' | 91 - '[' | 93 - ']' | 44 - ','
if string_on and ord(input_string[counter]) not in [34, 39]:
if ord(input_string[counter]) not in [91, 93, 58, 44]:
if ord(input_string[counter]) == 32 and not key_tmp.strip():
pass
else:
key_tmp += input_string[counter]
# edge case where there is quote around outer bracket
if ord(input_string[counter]) == 91 and string_on:
string_on = False
key_tmp = ""
# string markers of ' and "
if ord(input_string[counter]) in [34, 39]:
# insert new check if ' followed by 's'
exception_skip = False
if len(input_string) > counter+1:
if ord(input_string[counter+1]) in [115]:
exception_skip = True
# counter += 1
# end - new check
if not exception_skip:
if not string_on:
string_on = True
key_tmp = ""
else:
# end of string token
string_on = False
if len(key_tmp) > 0:
if not list_type:
if key_or_value == "key":
keys.append(key_tmp)
current_key = key_tmp
output_dict.update({current_key: []})
else:
values.append(key_tmp)
if current_key in output_dict:
output_dict[current_key].append(key_tmp)
else:
logger.warning("remediation - could not find key-value to correct - output "
"may be missing certain content in structured output.")
key_tmp = ""
else:
output_list.append(key_tmp)
values.append(key_tmp)
key_tmp = ""
if ord(input_string[counter]) == 58:
if len(input_string) > counter + 5:
for z in range(1, 5):
if ord(input_string[counter + z]) == 91:
key_or_value = "value"
counter += z - 1
break
if ord(input_string[counter]) == 93:
key_or_value = "key"
counter += 1
if counter >= len(input_string):
break
if not list_type:
# remediation successful in converting to dict output
if dedupe_values:
for keys, values in output_dict.items():
output_dict[keys] = list(set(values))
return response_type, output_dict
else:
# remediation successful in converting to list output
if dedupe_values:
dd_output = []
for elements in output_list:
if elements not in dd_output:
dd_output.append(elements)
# not using set because it can change the order of the list from output
# output_list = list(set(output_list))
output_list = dd_output
return response_type, output_list
def analyze_sampling(self,response):
""" Analyzes a llm response output dictionary and produces a 'sampling_stats' dictionary to provide
details on the effects, if any, of sampling in the output generation. """
sampling_stats = {}
if "logits" not in response or "output_tokens" not in response:
logger.warning("ModelCatalog - function get_fx_scores requires a response dictionary with 'logits' key - "
"not found in the current response provided. Set the model parameters to 'get_logits=True'"
"for function call to provide logits")
return sampling_stats
logits = response["logits"]
output_tokens = response["output_tokens"]
not_top_selected = 0
top_token_not_used = []
if len(output_tokens) == 0:
return sampling_stats
for x in range(0, len(output_tokens)):
top_selected = True
if output_tokens[x] != logits[x][0][0] and x > 0:
top_selected = False
top_token_not_used.append((x, output_tokens[x], logits[x]))
if not top_selected and x > 0:
not_top_selected += 1
tokens_considered = len(output_tokens) - 1
if tokens_considered > 0:
percent_top_token = (tokens_considered - not_top_selected) / tokens_considered
else:
percent_top_token = 0.0
# sampling_stats added to the output dictionary
sampling_stats.update({"total_output_tokens": len(output_tokens),
"percent_top_token": round(percent_top_token, 3),
"not_top_tokens": top_token_not_used})
return sampling_stats
def get_fx_scores(self,response, model_name, top_choices=3, logit_count=1, api_key=None):
""" Provides useful metrics and scores derived from analyzing the logits and output tokens from function call
llm response - currently only supported for HFGenerative and GGUFGenerative models.
Inputs:
-- llm response dictionary, including logits and output token
-- model_name which will be used to lookup the model card and get applicable tokenizer(s)
-- tokenizer will be used to decode output tokens, logits and identify key
'value zone' markers for the output response, e.g., identify list boundaries '[' and ']'
-- top_choices - number of candidates to consider in each logit, e.g., top 3 choices considered
-- logit_count - number of tokens to consider in the value zone, whether the first only, or more
-- api_key - optional, if tokenizer in private repository requiring an api key
Output (dictionary):
-- for each key in the output response, there is a list of the candidate logits in the value zone associated
with that key - the list will be the length of the logit count requested
-- a sampling_stats key will also be produced that will provide summary data on the number of 'value zone'
tokens, the percentage taken from the top output logit candidate and a list of the 'sampled', e.g.,
'not top' logits taken
"""
# model name - look up model card
model_card = self.lookup_model_card(model_name)
hf_tokenizer_name = None
tokenizer_local = None
if "tokenizer" in model_card:
hf_tokenizer_name = model_card["tokenizer"]
if "tokenizer_local" in model_card:
tokenizer_local = model_card["tokenizer_local"]
# output is a dict of dict
output = {}
if "logits" not in response or "output_tokens" not in response:
logger.warning("ModelCatalog - function get_fx_scores requires a response dictionary with 'logits' key - "
"not found in the current response provided. Set the model parameters to 'get_logits=True'"
"for function call to provide logits")
return output
logits = response["logits"]
keys_list = []
llm_response = response["llm_response"]
if isinstance(llm_response, dict):
for key, value in llm_response.items():
keys_list.append(key)
elif isinstance(llm_response, list):
keys_list.append("llm_response")
else:
keys_list.append("llm_response")
# tokenizer load
if tokenizer_local:
tokenizer = LocalTokenizer(tokenizer_fn=model_card["tokenizer_local"])
elif hf_tokenizer_name and util.find_spec("transformers"):
# hf tokenizer name
pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=True, custom_loader=None)
tokenizer = pt_loader.get_tokenizer(hf_tokenizer_name)
else:
raise LLMWareException(message="Exception: could not identify tokenizer to use")
vz_choices = []
vz_capture_on = False
key_counter = 0
min_threshold = 0.005
vz_logits = 0
vz_top_logits = 0
top_token_not_used = []
for i, toks in enumerate(response["output_tokens"]):
decoded = tokenizer.decode(toks)
if "]" in decoded:
vz_capture_on = False
if vz_choices:
output.update({keys_list[key_counter]: vz_choices})
key_counter += 1
vz_choices = []
if vz_capture_on:
new_entry = {}
if toks == logits[i][0][0]:
vz_top_logits += 1
else:
# the output token does not correspond to the logit with the highest score, so there was a
# 'sampling' effect to this generation - adding this token and corresponding logit to be saved
# and provided as output in 'sampling_stats'
top_token_not_used.append((i, toks, logits[i]))
vz_logits += 1
for x in range(0, top_choices):
if logits[i][x][1] >= min_threshold:
new_entry.update({tokenizer.decode(logits[i][x][0]): round(logits[i][x][1], 3)})
if len(vz_choices) < logit_count:
vz_choices.append(new_entry)
if "[" in decoded:
vz_capture_on = True
vz_choices = []
if vz_top_logits > 0:
top_token_in_value_zone = round(vz_logits / vz_top_logits, 2)
else:
top_token_in_value_zone = 0.0
# sampling_stats added to the output dictionary
output.update({"sampling_stats": {"total_vz_tokens": vz_logits,
"percent_top_token": top_token_in_value_zone,
"not_top_tokens": top_token_not_used}
})
return output
def gpu_available(self, suppress_warnings=True, driver_min_levels=None):
""" Checks if CUDA GPU drivers found on machine, and whether the drivers are at
the required minimum level.
-- driver_min_level is a tuple of integers consisting of the major/minor driver level, e.g., (525, 15)
-- if no driver_min_level is passed, then the test will be skipped and come back False by default.
"""
major_driver = 0
minor_driver = 0
result = {"gpu_found": False, "drivers_current": False,
"gpu_name": "", "driver": "", "multiple_gpu": False}
try:
from subprocess import Popen, PIPE
except:
if not suppress_warnings:
logger.warning("ModelCatalog - check gpu availability - unable to check if gpu available")
return result
if sys.platform.lower() == "win32":
nvidia_smi = shutil.which('nvidia-smi')
elif sys.platform.lower().startswith("linux"):
nvidia_smi = "nvidia-smi"
else:
if not suppress_warnings:
logger.warning("ModelCatalog - check gpu availability - only check for CUDA drivers on Windows or Linux")
return result
try:
gpu_pipe = Popen([nvidia_smi, "--query-gpu=index,driver_version,name","--format=csv,noheader,nounits"],
stdout=PIPE)
gpu, errors = gpu_pipe.communicate()
except Exception as e:
gpu = []
errors = e
if gpu:
result["gpu_found"] = True
# only looking at 'first' gpu
results = str(gpu).split(",")
if len(results) > 1:
#TODO: handle multiple GPUs on device!
driver_index = results[0].strip().encode('utf')
driver_level = results[1].strip()
result["driver"] = driver_level
if len(results) > 2:
result["gpu_name"] = results[2].strip()
if driver_min_levels:
driver_split = driver_level.split(".")
if len(driver_split) > 0:
try:
major_driver = int(driver_split[0].strip())
if len(driver_split) > 1:
minor_driver = int(driver_split[1].strip())
except:
pass
if major_driver > driver_min_levels[0] or (major_driver == driver_min_levels[0]
and minor_driver >= driver_min_levels[1]):
result["drivers_current"] = True
else:
result["drivers_current"] = False
logger.warning(f"ModelCatalog - check gpu availability - CUDA device found - but drivers "
f"look out of date, relative to required min levels: \n"
f"--drivers found: {driver_level}\n"
f"--min required: {driver_min_levels}\n")
return result
class PromptCatalog:
""" PromptCatalog manages prompt styles and prompt wrappers and builds prompt templates for inference
generation. """
def __init__(self):
self.prompt_catalog = global_default_prompt_catalog
self.prompt_wrappers = _ModelRegistry().prompt_wrappers
self.prompt_wrapper_lookup = _ModelRegistry().get_wrapper_list()
self.prompt_list = self.list_all_prompts()
def lookup_prompt(self, prompt_name):
""" Looks up a predefined prompt template by prompt_name. """
for prompts in self.prompt_catalog:
if prompts["prompt_name"] == prompt_name:
return prompts
return None
def get_all_prompts(self):
""" Returns all predefined prompts. """
return self.prompt_catalog
def list_all_prompts(self):
""" Returns a list of all predefined prompts. """
prompt_list = []
for prompt in self.prompt_catalog:
if "prompt_name" in prompt:
prompt_list.append(prompt["prompt_name"])
return prompt_list
def parse_instruction_for_user_vars(self, prompt_card, inference_dict=None):
""" Utility method that looks for user_vars in prompt card to dynamically insert into Prompt. """
# if no user vars key in prompt_card, then return instruction unchanged
if "user_vars" not in prompt_card:
return prompt_card["instruction"]
if not prompt_card["user_vars"]:
return prompt_card["instruction"]
# if no inference_dict, then define as empty dictionary
if not inference_dict:
inference_dict = {}
# in this case, will 'parameterize' and dynamically update instruction
tokens = prompt_card["instruction"].split(" ")
updated_instruction = ""
for i, t in enumerate(tokens):
if t.startswith("{{") and t.endswith("}}"):
t_core = t[2:-2]
# if value found for key in the inference dict, then apply as true 'user_vars'
if t_core in inference_dict:
new_inserted_token = inference_dict[t_core]
updated_instruction += str(new_inserted_token) + " "
else:
# apply default value found in the prompt card as back-up
if t_core in prompt_card["user_vars"]:
new_inserted_token = prompt_card["user_vars"][t_core]
updated_instruction += str(new_inserted_token) + " "
else:
updated_instruction += t + " "
logger.debug(f"PromptCatalog - constructed dynamic instruction - {updated_instruction}")
return updated_instruction.strip()
def build_core_prompt(self, prompt_card=None, prompt_name=None, separator="\n", query=None, context=None,
inference_dict=None):
""" Builds the core prompt from the prompt_card template. """
if not context: context = ""
if not query: query = ""
if not prompt_card and not prompt_name:
# error - returning query
logger.warning("PromptCatalog - no prompt selected in PromptCatalog().build_core_prompt")
prompt_dict = {"core_prompt": context + "\n" + query, "prompt_card": {}}
return prompt_dict
if not prompt_card:
prompt_card = PromptCatalog().lookup_prompt(prompt_name)
logger.debug(f"PromptCatalog - prompt_card - {prompt_card}")
core_prompt = ""
if prompt_card:
for keys in prompt_card["run_order"]:
if keys == "instruction":
# special handler
instruction = self.parse_instruction_for_user_vars(prompt_card, inference_dict=inference_dict)
core_prompt += instruction + separator
else:
if not keys.startswith("$"):
core_prompt += prompt_card[keys] + separator
else:
if keys == "$query":
core_prompt += query + separator
if keys == "$context":
core_prompt += context + separator
# update instruction, if user_vars accepted in instruction
"""
if "instruction" in prompt_card:
prompt_card["instruction"] = self.parse_instruction_for_user_vars(prompt_card,inference_dict=inference_dict)
core_prompt += prompt_card["instruction"]
"""
prompt_dict = {"core_prompt": core_prompt, "prompt_card": prompt_card}
logger.debug(f"PromptCatalog - prompt created - {prompt_dict}")
return prompt_dict
def add_custom_prompt_card(self, prompt_name, run_order_list, prompt_dict, prompt_description=None):
""" Registers a new custom prompt_card with 'run_order_list' that shows how to assemble the components
of a Prompt. """
new_prompt_card = {"prompt_name": prompt_name,
"prompt_description": prompt_description,
"run_order": run_order_list}
for keys, values in prompt_dict.items():
new_prompt_card.update({keys: values})
self.prompt_catalog.append(new_prompt_card)
return new_prompt_card
def apply_prompt_wrapper(self, text, prompt_wrapper,
separator="\n",
instruction=None,
chat_history=None):
""" Applies the selected prompt_wrapper to the prompt. """
output_text = text
if prompt_wrapper not in self.prompt_wrappers:
logger.info(f"PromptCatalog - apply_prompt_wrapper - selected wrapper - {prompt_wrapper} - could not be identified - "
f"returning text prompt without any special format wrapping")
return output_text
if prompt_wrapper == "chatgpt":
return self.wrap_chatgpt_sample(text, instruction)
else:
wrapped_prompt = self.wrap_custom(text, prompt_wrapper,
instruction=instruction,
chat_history=chat_history)
return wrapped_prompt
def wrap_custom(self, text, wrapper_type, chat_history=None,
instruction=None):
""" Provides option for chat history, packaged as a list of 'turns'
with each turn consisting of two dictionary entries -
'user' and 'assistant' """
#TODO: apply safeguards to max output
prompt_out = ""
if wrapper_type in self.prompt_wrapper_lookup:
prompt_template = self.prompt_wrapper_lookup[wrapper_type]
if "system_start" in prompt_template:
if prompt_template["system_start"] != "":
prompt_out += prompt_template["system_start"]
if instruction:
prompt_out += instruction
else:
prompt_out += "You are a helpful assistant."
if "system_stop" in prompt_template:
prompt_out += prompt_template["system_stop"]
if chat_history:
for turn in chat_history:
# user part of turn
if "main_start" in prompt_template:
prompt_out += prompt_template["main_start"]
prompt_out += turn["user"]
if "main_stop" in prompt_template:
prompt_out += prompt_template["main_stop"]
# assistant part of turn
if "start_llm_response" in prompt_template:
prompt_out += prompt_template["start_llm_response"]
prompt_out += turn["assistant"]
if "main_start" in prompt_template:
prompt_out += prompt_template["main_start"]
prompt_out += text
if "main_stop" in prompt_template:
prompt_out += prompt_template["main_stop"]
if "start_llm_response" in prompt_template:
prompt_out += prompt_template["start_llm_response"]
else:
prompt_out = text
return prompt_out
def wrap_chatgpt_sample(self, text, instruction):
""" Applies chatgpt format wrapper to a prompt. """
if not instruction:
instruction = "You are a helpful assistant."
new_sample = [{"role": "system", "content": instruction},
{"role": "user", "content": text}]
return new_sample
class InferenceHistory:
""" Global State History of All Inferences Completed in Session """
base_model_keys = ["llm_response", "usage", "logits", "output_tokens", "prompt", "add_context","final_prompt",
"model_name", "model_card", "temperature", "add_prompt_engineering",
"model_class", "model_category", "prompt_wrapper", "time_stamp"
]
inference_history = []
global_inference_counter = 0
save = True
@classmethod
def get_base_model_keys(cls):
return cls.base_model_keys
@classmethod
def add_base_model_key(cls, new_key):
if new_key not in cls.base_model_keys:
cls.base_model_keys.append(new_key)
return True
@classmethod
def del_base_model_key(cls, key_to_delete):
if key_to_delete in cls.base_model_keys:
del cls.base_model_keys[key_to_delete]
return True
@classmethod
def get_transactions(cls):
""" List current view of implemented supported vector db for embeddings. """
return cls.inference_history
@classmethod
def add_transaction(cls, model_state_dict):
""" Adds a vector db including the module and class. """
cls.inference_history.append(model_state_dict)
return True
@classmethod
def get_global_inference_count(cls):
return cls.global_inference_counter
@classmethod
def increment_global_inference_count(cls):
cls.global_inference_counter += 1
return cls.global_inference_counter
@classmethod
def reset_global_inference_count(cls):
cls.global_inference_counter = 0
return cls.global_inference_counter
@classmethod
def get_save_status(cls):
return cls.save
@classmethod
def set_save_status(cls, status):
if isinstance(status, bool):
cls.save = status
else:
raise LLMWareException(message="Exception: save status must be boolean - True/False")
def register(kv_dict):
""" Default register function called after each Model inference activity. This method can be over-ridden and
customized by re-routing the LLMWareConfig as follows:
`LLMWareConfig().set_config('model_register', {'module': 'my_module', 'class': 'my_register_fx'})
`module` currently points to this module: 'llmware.models'
`class` currently points to this method: 'register'
"""
# if save status set to False, then skip
if not InferenceHistory().get_save_status():
logger.debug(f"InferenceHistory - skipping registration since save status is False")
return True
for k, v in kv_dict.items():
logger.debug(f"InferenceHistory - register: {k} - {v}")
InferenceHistory().increment_global_inference_count()
logger.debug(f"InferenceHistory - global inference counter - {InferenceHistory().get_global_inference_count()}")
# by default, will register all generative inferences, but takes no action to track embedding inferences
if "model_category" in kv_dict:
if kv_dict["model_category"] == "generative":
InferenceHistory().add_transaction(kv_dict)
return True
def post_init(kv_dict):
""" Not implemented by default. """
logger.debug(f"Model Load - in post_init - not implemented - returning True - no action taken")
return True
def validate(kv_dict):
""" Not implemented by default. """
logger.debug(f"Model Load - validate - not implemented - returning True - no action taken")
return True
def preview(kv_dict):
""" Not implemented by default. """
logger.debug(f"Model Load - preview - not implemented - returning True - no action taken")
return True
def route_optimizer(kv_dict):
""" Not implemented by default. """
logger.debug(f"Model Route Optimizer - not implemented - returning True - no action taken")
return True
class BaseModel:
""" BaseModel class subclassed by all models. Should not be instantiated directly. Provides several
common utility methods across each of the Model class implementations. """
def __init__(self, **kwargs):
# InferenceHistory provides a set of state parameters to be captured from each Model instantiation
self.base_model_keys = InferenceHistory().get_base_model_keys()
self.time_stamp = None
self.model_class = None
self.model_category = None
self.model_card = {}
self.tokenizer = None
self.URL_BASE = None
self.api_endpoint = None
self.unlock_on_completion = None
# parameters moved to base model
self.separator = "\n"
self.instruction_following = False
self.prompt_wrapper = None
self.add_prompt_engineering = True
# output inference parameters
for keys in self.base_model_keys:
if keys in kwargs:
setattr(self, keys, kwargs[keys])
else:
setattr(self, keys, None)
def to_state_dict(self):
""" Writes selected model state parameters to dictionary. """
state_dict = {}
for keys in self.base_model_keys:
if hasattr(self, keys):
state_dict.update({keys: getattr(self, keys)})
return state_dict
def load_model_for_inference(self, loading_instructions):
# not implemented in base model
pass
def method_resolver(self, config_name):
""" Resolves method to invoke selected function. """
process_class = ""
process_method = ""
method_exec = None
state_dict = self.to_state_dict()
process = LLMWareConfig().get_config(config_name)
process_module = process["module"]
if "class" in process:
process_class = process["class"]
if "method" in process:
process_method = process["method"]
module_exec = importlib.import_module(process_module)
if process_class:
if hasattr(module_exec, process_class):
class_exec = getattr(module_exec, process_class)()
if process_method:
if hasattr(class_exec, process_method):
method_exec = getattr(class_exec, process_method)
else:
if hasattr(module_exec, process_method):
method_exec = getattr(module_exec, process_method)
if method_exec:
success = method_exec(state_dict)
if isinstance(success, dict):
# write attributes, if any, to the Model instance state
for k, v in success.items():
setattr(self, k, v)
return True
def set_api_key(self, api_key, env_var="USER_MANAGED_API_KEY"):
""" Sets the API key - generally not needed for self-hosted models. """
os.environ[env_var] = api_key
logger.info("BaseModel - added and stored api_key in environmental "
"variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("BaseModel - _get_api_key could not successfully "
"retrieve value from: %s ", env_var)
return self.api_key
def post_init(self):
return self.method_resolver("model_post_init")
def register(self):
if self.unlock_on_completion:
ModelResources().unlock(self.unlock_on_completion)
return self.method_resolver("model_register")
def validate(self):
return self.method_resolver("model_validate")
def preview(self):
return self.method_resolver("model_preview")
def _lookup_endpoint(self, api_name, api_catalog):
""" Internal lookup utility to pull api card. """
for entries in api_catalog:
if entries["api_name"] == api_name:
return entries
return {}
def prune_context(self, ctx, front=100,back=100):
# apply pruning of stop words
pruned_ctx = Utilities().prune_stop_words(ctx,front=front,back=back)
# test len
pruned_tokens = self.count_tokens(pruned_ctx)
logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}")
# extra pruning for very large contexts
# need to reduce for 14B parameter models
if pruned_tokens > 16000:
start = pruned_ctx[0:1000]
end = pruned_ctx[pruned_tokens-5000:]
super_pruned = start + end
pruned_tokens = self.count_tokens(super_pruned)
logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}")
pruned_ctx = super_pruned
return pruned_ctx
def count_tokens(self, ctx, tokenizer=None):
if not tokenizer:
tokenizer = self.tokenizer
toks = tokenizer.encode(ctx)
tok_len = len(toks.ids)
return tok_len
def prompt_engineer(self, query, context, inference_dict):
""" Applies prompt and templating preparation. """
# adding chat history to inference_dict handler
chat_history = None
system_instruction = None
if inference_dict:
if "system_instruction" in inference_dict:
system_instruction = inference_dict["system_instruction"]
if "chat_history" in inference_dict:
chat_history = inference_dict["chat_history"]
if self.instruction_following:
logger.info(f"BaseModel - prompt_engineer - found deprecated setting - "
f"instruction_following set to True - may cause unpredictable results.")
# self.instruction_following = False
# if loaded model was not pretrained on instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output,
self.prompt_wrapper,
chat_history=chat_history,
instruction=system_instruction)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def function_call(self, context, function=None, params=None, get_logits=False,
temperature=-99, max_output=None):
""" This is the key inference method for SLIM models - takes a context passage and a key list
which is packaged in the prompt as the keys for the dictionary output"""
output_response = {}
return output_response
def fc_prompt_engineer(self, context, params=None, function=None,
trailing_space= ""):
""" Prompt engineering for Function Call prompts. """
# prepare SLIM prompt
class_str = ""
for key in params:
class_str += str(key) + ", "
if class_str.endswith(", "):
class_str = class_str[:-2]
f = str(function)
# key templating format for SLIM function calls
full_prompt = "<human>: " + context + "\n" + "<{}> {} </{}>".format(f, class_str, f) + "\n<bot>:"
full_prompt = full_prompt + trailing_space
return full_prompt
def close(self):
""" General purpose 'close' method with any special wind-down
procedures at the time of closing out an inferencing session. """
pass
class ONNXGenerativeModel(BaseModel):
"""ONNXGenerativeModel class implements the ONNX Runtime generative model interface, and is used generally for
models converted from Pytorch into ONNX for faster inference performance and packaging on Windows platforms
and x86 architectures. """
def __init__(self, model_name=None, api_key=None, model_card=None, instruction_following=False, context_window=2048,
sample=True, max_output=100, temperature=0.3, get_logits=False, api_endpoint=None, **kwargs):
super().__init__()
self.model_class = "ONNXGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = None
self.tokenizer = None
self.generator = None
self.context_window = context_window
self.sample = sample
self.get_logits = get_logits
self.auto_remediate_function_call_output = True
# Function Call parameters
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
self.tool_type = None
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
if "context_window" in model_card:
self.context_window = model_card["context_window"]
# insert dynamic onnx load here
if not api_endpoint:
global GLOBAL_ONNX_GENAI_RUNTIME
if not GLOBAL_ONNX_GENAI_RUNTIME:
if util.find_spec("onnxruntime_genai"):
try:
global og
og = importlib.import_module("onnxruntime_genai")
GLOBAL_ONNX_GENAI_RUNTIME = True
except:
raise LLMWareException(message="ONNXGenerativeModel: could not load onnxruntime_genai module. "
"If you have pip installed the library, then please check "
"that your platform is supported by onnxruntime.")
else:
import platform
if platform.system() == "Darwin":
raise LLMWareException(message=f"ONNXGenerativeModel: identified current platform as 'Mac OS' "
f"which is not supported for onnxruntime_genai currently. "
f"\nWe would recommend using GGUF for generative inference on a "
f"Mac, or if you wish to use ONNXGenerativeModel, then please "
f"shift to a supported Windows or Linux platform.")
raise LLMWareException(message="ONNXGenerativeModel: need to import "
"onnxruntime_genai to use this class, e.g., 'pip3 install "
"onnxruntime_genai`")
# end dynamic import here
if model_name and not api_endpoint:
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(self.model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
self.hf_tokenizer_name = hf_repo_name
self.model = None
self.tokenizer = None
self.tokenizer_stream = None
# this can be over-ridden post initiation if needed for custom models
self.prompt_wrapper = "human_bot"
self.instruction_following = False
self.params = None
# set specific parameters associated with custom models
# note - these two parameters will control how prompts are handled - model-specific
self.prompt_wrapper = "human_bot"
self.instruction_following = instruction_following
if not model_card:
# safety - empty iterable rather than 'None'
model_card = {}
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
# sets trailing space default when constructing the prompt
# in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves
# performance
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = self.context_window
self.max_input_len = int(0.5 * self.context_window)
self.llm_max_output_len = int(0.5 * self.context_window)
# key output parameters
self.max_output = max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# use 0 as eos token id by default in generation -> but try to pull from model config
self.eos_token_id = 0
# will load model and inference onto gpu,
# if (a) CUDA available and (b) use_gpu_if_available set to True (default)
# TODO: CUDA option handling for ONNX models
if not api_endpoint:
self.use_gpu = False
else:
self.use_gpu = False
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load ONNX model."
# temperature settings
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.3
self.temperature = 0.3
self.add_prompt_engineering = False
self.add_context = ""
self.context = ""
self.prompt = ""
self.api_endpoint = api_endpoint
self.model_repo_path = None
self.post_init()
def load_model_for_inference(self, loading_directions, model_card=None):
""" Loads ONNX Model from local path using loading directions. """
global og
self.model_repo_path = loading_directions
if model_card:
self.model_card = model_card
self.validate()
onnx_model_path = os.path.join(LLMWareConfig().get_model_repo_path(),
self.model_name)
try:
self.model = og.Model(onnx_model_path)
self.tokenizer = og.Tokenizer(self.model)
self.tokenizer_stream = self.tokenizer.create_stream()
except:
raise LLMWareException(message=f"ONNXGenerativeModel - unable to load and instantiate the model at: "
f"\n{onnx_model_path}\nThis could be for a number of reasons, but "
f"most likely is one of the following:"
f"\n1. onnxruntime not installed correctly."
f"\n2. platform (e.g, Mac) is not supported by current ONNX Build."
f"\n3. model could not be found at this path, or is not a valid ONNX model."
)
search_options = {}
# max length set at minimum of 2048
# adjusted to the actual model context window (if available)
# currently cap at 'safety' max of 8192
# --seems to have performance impact at larger lengths
max_length = max(2048, self.max_total_len)
if max_length > 8192:
max_length = 8192
search_options['max_length'] = max_length
self.params = og.GeneratorParams(self.model)
self.params.set_search_options(**search_options)
return self
def unload_model(self):
""" Remove model pointer from memory space. In most use cases, simply deleting the model pointer will suffice
to trigger Python memory cleanup with an explicit call to gc.collect(). This is WIP and will continue
to test different scenarios to explore the best 'safe' unload steps. """
self.model = None
self.tokenizer = None
import gc
gc.collect()
return True
def set_api_key(self, api_key, env_var="USER_MANAGED_ONNX_API_KEY"):
""" Sets the API key - generally not needed for ONNX self-hosted models. """
os.environ[env_var] = api_key
logger.info("ONNXGenerativeModel - added and stored ONNX api_key in "
"environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_ONNX_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("ONNXGenerativeModel - _get_api_key could not successfully "
"retrieve value from: %s ", env_var)
return self.api_key
def token_counter(self, text_sample):
""" Not Used for ONNXGenerativeModel class - Quick approximate token counter -
uses default tokenizer so may have minor differences from the model's actual tokenization. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer(self, query, context, inference_dict):
""" Applies prompt and templating preparation. """
# if loaded model was not pretrained to require instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper,
instruction=None)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None):
""" Executes generation inference on model. """
from llmware.configs import ONNXConfig
legacy = ONNXConfig().get_legacy_flag()
global og
# first prepare the prompt
t0 = time.time()
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
# show warning if function calling model
if self.fc_supported:
logger.warning("ONNXGenerativeModel - this is a function calling model - using .inference may lead to "
"unexpected results. Recommended to use the .function_call method to ensure correct prompt "
"template packaging.")
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# text_prompt = prompt_final + "\n"
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
text_prompt = prompt_final + self.trailing_space
input_tokens = self.tokenizer.encode(text_prompt)
if legacy:
self.params.input_ids = input_tokens
token_count = 0
output = ""
try:
generator = og.Generator(self.model, self.params)
except:
raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with "
f"model and prompt failed. This is most likely due to an error in the "
f"installation of the onnxruntime, or a problem with loading either the "
f"model or the input tokens.")
# borrow 'get_first_token_speed' config from GGUFConfigs
get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed")
t_gen_start = time.time()
first_token_processing_time = -1.0
if not legacy:
generator.append_tokens(input_tokens)
while not generator.is_done():
token_count += 1
if legacy:
generator.compute_logits()
generator.generate_next_token()
# get logits - in most cases, get_logits is set to False for basic inference
if self.get_logits:
logit = generator.get_output("logits")
self.register_top_logits(logit)
new_token = generator.get_next_tokens()[0]
# first token capture
if get_first_token_speed:
if token_count == 1:
first_token_processing_time = time.time() - t_gen_start
# first token capture ends here
if self.get_logits:
self.output_tokens.append(new_token)
output += self.tokenizer_stream.decode(new_token)
# add stream on/off options
# print(self.tokenizer_stream.decode(new_token), end="", flush=True)
if token_count > self.max_output:
break
# direct deletion of generator recommended in onnxruntime_genai examples
del generator
llm_response = {"llm_response": output, "usage": {}}
usage = {"input": len(input_tokens),
"output": token_count,
"total": len(input_tokens) + token_count,
"metric": "tokens",
"processing_time": time.time() - t0}
if get_first_token_speed:
usage.update({"first_token_processing_time": first_token_processing_time})
output_response = {"llm_response": output, "usage": usage}
if self.get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def fc_prompt_engineer(self, context, params=None, function=None):
""" Prompt engineering for Function Call prompts. """
if not params:
params = self.primary_keys
if not function:
function = self.function[0]
# prepare SLIM prompt
class_str = ""
for key in params:
class_str += str(key) + ", "
if class_str.endswith(", "):
class_str = class_str[:-2]
f = str(function)
# key templating format for SLIM function calls
full_prompt = "<human>: " + context + "\n" + "<{}> {} </{}>".format(f, class_str, f) + "\n<bot>:"
full_prompt = full_prompt + self.trailing_space
return full_prompt
def register_top_logits(self, logit):
""" Gets the top logits and keeps a running log for output analysis. """
# logit will be in form of (1,1,vocab_len), for all but the first logit
# if first logit (will have shape of context len - add [-1])
if logit.shape[1] > 1:
# used for first logit with shape, e.g., (1,input_token_len,vocab_size)
logit_array = logit.squeeze()[-1]
else:
# all other logits after the first token
logit_array = logit.squeeze()
logit_size = logit.shape[-1]
# useful check on shape of logit_array
logit_array_size = logit_array.shape
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
for x in range(0, self.top_logit_count):
# round the float number to 3 digits
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3))
top_logits.append(pair)
self.logits_record.append(top_logits)
return top_logits
def function_call(self, context, function=None, params=None, get_logits=True,
temperature=-99, max_output=None):
""" This is the key inference method for SLIM models - takes a context passage and a key list
which is packaged in the prompt as the keys for the dictionary output"""
from llmware.configs import ONNXConfig
legacy = ONNXConfig().get_legacy_flag()
t0 = time.time()
self.context = context
if not self.fc_supported:
logger.warning(f"ONNXGenerativeModel - loaded model does not support function calls. "
"Please either use the standard .inference method with this model, or use a "
"model that has 'function_calls' key set to True in its model card.")
return []
# reset and start from scratch with new function call
self.output_tokens = []
self.logits_record = []
if temperature != -99:
self.temperature = temperature
if max_output:
self.target_requested_output_tokens = max_output
if get_logits:
self.get_logits = get_logits
if params:
self.primary_keys = params
if function:
self.function = function
if not self.primary_keys:
logger.warning(f"ONNXGenerativeModel - function call - no keys provided - "
f"function call may yield unpredictable results")
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.function_call_over_api_endpoint(model_name=self.model_name,
context=self.context, params=self.primary_keys,
function=self.function,
api_key=self.api_key, get_logits=self.get_logits)
# END - route to api endpoint
prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=self.function)
input_tokens = self.tokenizer.encode(prompt)
if legacy:
self.params.input_ids = input_tokens
token_count = 0
output = ""
try:
generator = og.Generator(self.model, self.params)
except:
raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with "
f"model and prompt failed. This is most likely due to an error in the "
f"installation of the onnxruntime, or a problem with loading either the "
f"model or the input tokens.")
if not legacy:
generator.append_tokens(input_tokens)
while not generator.is_done():
token_count += 1
if legacy:
generator.compute_logits()
# to get logit value
if self.get_logits:
logit = generator.get_output("logits")
self.register_top_logits(logit)
generator.generate_next_token()
new_token = generator.get_next_tokens()[0]
if self.get_logits:
self.output_tokens.append(new_token)
output += self.tokenizer_stream.decode(new_token)
# add as streaming option to turn on/off
# print(self.tokenizer_stream.decode(new_token), end="", flush=True)
if token_count >= self.max_output:
break
# done with generator
del generator
llm_response = {"llm_response": output, "usage": {}}
usage = {"input": len(input_tokens),
"output": token_count,
"total": len(input_tokens) + token_count,
"metric": "tokens",
"processing_time": time.time() - t0}
output_response = {"llm_response": output, "usage": usage}
# end - post-processing
try:
import ast
output_value = ast.literal_eval(output)
output_type = "dict"
# allow for multiple valid object types - will expand over time
if isinstance(output_value, dict): output_type = "dict"
if isinstance(output_value, list): output_type = "list"
usage.update({"type": output_type})
except:
# could not convert automatically to python object
output_type = "string"
usage.update({"type": output_type})
output_value = output
# INSERT NEW HERE
if self.auto_remediate_function_call_output:
# attempt to remediate
output_type, output_rem = ModelCatalog().remediate_function_call_string(output)
usage.update({"type": output_type, "remediation": True})
output_value = output_rem
if output_type == "string":
logger.warning(f"ONNXGenerativeModel - function call - automatic conversion of function call output "
f"failed, and attempt to remediate was not successful - {output}")
else:
logger.info(f"ONNXGenerativeModel - function call output could not be automatically converted, but "
f"remediation was successful to type -{output_type}")
# INSERT ENDS HERE
output_response = {"llm_response": output_value, "usage": usage}
if get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output_value
self.usage = usage
self.final_prompt = prompt
self.register()
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None):
""" Executes stream generation inference on model. """
from llmware.configs import ONNXConfig
legacy = ONNXConfig().get_legacy_flag()
# first prepare the prompt
t0 = time.time()
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
# show warning if function calling model
if self.fc_supported:
logger.warning("ONNXGenerativeModel - this is a function calling model - "
"using .inference may lead to unexpected "
"results. Recommended to use the .function_call method to "
"ensure correct prompt template packaging.")
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# text_prompt = prompt_final + "\n"
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
text_prompt = prompt_final + self.trailing_space
input_tokens = self.tokenizer.encode(text_prompt)
if legacy:
self.params.input_ids = input_tokens
token_count = 0
output = ""
# adding as a state var so it can be shut down by chat app if user terminates
try:
self.generator = og.Generator(self.model, self.params)
except:
raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with "
f"model and prompt failed. This is most likely due to an error in the "
f"installation of the onnxruntime, or a problem with loading either the "
f"model or the input tokens.")
if not legacy:
self.generator.append_tokens(input_tokens)
while not self.generator.is_done():
token_count += 1
if legacy:
self.generator.compute_logits()
self.generator.generate_next_token()
self.get_logits = False
# to get logit value
if self.get_logits:
logit = self.generator.get_output("logits")
self.register_top_logits(logit)
new_token = self.generator.get_next_tokens()[0]
if self.get_logits:
self.output_tokens.append(new_token)
output += self.tokenizer_stream.decode(new_token)
if token_count > self.max_output:
break
yield self.tokenizer_stream.decode(new_token)
print()
# del self.generator
self.generator = None
llm_response = {"llm_response": output, "usage": {}}
usage = {"input": len(input_tokens),
"output": token_count,
"total": len(input_tokens) + token_count,
"metric": "tokens",
"processing_time": time.time() - t0}
output_response = {"llm_response": output, "usage": usage}
if self.get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def cleanup_stream_gen_on_early_stop(self):
""" Utility method to call if streaming interrupted early to clean up the generator. """
self.generator = None
return True
def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False):
""" Called by .inference method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
import ast
import requests
self.prompt = prompt
self.context = context
self.preview()
url = self.api_endpoint + "{}".format("/")
output_raw = requests.post(url, data={"model_name": self.model_name,
"question": self.prompt,
"context": self.context,
"api_key": self.api_key,
"max_output": self.max_output,
"temperature": self.temperature})
try:
output = json.loads(output_raw.text)
# will attempt to unpack logits - but catch any exceptions and skip
if "logits" in output:
try:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
except:
output["logits"] = []
# will attempt to unpack output tokens - but catch any exceptions and skip
if "output_tokens" in output:
try:
# alt: ot_int = [int(x) for x in output["output_tokens"]]
# alt: output["output_tokens"] = ot_int
output_tokens = ast.literal_eval(output["output_tokens"])
output["output_tokens"] = output_tokens
except:
output["output_tokens"] = []
except:
logger.warning("warning: api inference was not successful")
output = {"llm_response": "api-inference-error", "usage": {}}
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="",
function=None, endpoint_base=None, api_key=None, get_logits=False):
""" Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
# send to api agent server
import ast
import requests
self.context = context
self.tool_type = tool_type
if model_name:
self.model_name = model_name
self.preview()
if endpoint_base:
self.api_endpoint = endpoint_base
if api_key:
# e.g., "demo-test"
self.api_key = api_key
if not params:
model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type]
mc = ModelCatalog().lookup_model_card(model_name)
if "primary_keys" in mc:
params = mc["primary_keys"]
self.primary_keys = params
if function:
self.function = function
self.context = context
self.prompt = prompt
url = self.api_endpoint + "{}".format("/agent")
output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key,
"tool_type": self.tool_type,
"function": self.function,
"params": self.primary_keys, "max_output": 50,
"temperature": 0.0, "sample": False, "prompt": self.prompt,
"context": self.context, "get_logits": True})
try:
output = json.loads(output_raw.text)
if "logits" in output:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
if "output_tokens" in output:
ot_int = [int(x) for x in output["output_tokens"]]
output["output_tokens"] = ot_int
# need to clean up logits
except:
logger.warning(f"ONNXGenerativeModel - function call - api inference was not successful")
output = {}
logger.info(f"ONNXGenerativeModel - executed Agent call over API endpoint - "
f"{self.model_name} - {self.function} - {output}")
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
class OVGenerativeModel(BaseModel):
""" OVGenerativeModel class implements the OpenVino generative model interface for fast inference
performance on x86 Intel architectures, including both Intel CPU and GPU. """
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
prompt_wrapper=None, instruction_following=False, context_window=2048,
sample=False,max_output=100, temperature=0.0,
get_logits=False, api_endpoint=None, device="GPU",
pipeline="text2text", **kwargs):
super().__init__()
self.model_class = "OVGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = model
self.tokenizer = tokenizer
self.sample=sample
self.get_logits=get_logits
self.pipeline = pipeline
if get_logits:
logger.warning(f"OVGenerativeModel - current implementation does not support "
f"get_logits option.")
self.get_logits = False
self.auto_remediate_function_call_output = True
# Function Call parameters
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
self.cache_dir = None
self.device = device
if "device" in kwargs:
self.device = kwargs["device"]
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
# will look for special cache_dir set in the model card
# can be over-ridden if passed as kwarg in loading model
if "cache_dir" in model_card:
self.cache_dir = model_card["cache_dir"]
if "pipeline" in model_card:
self.pipeline = model_card["pipeline"]
# will auto-detect NPU model and set device accordingly
if "npu_optimized" in model_card:
self.device = "NPU"
# insert dynamic openvino load here
if not api_endpoint:
global openvino
global ovg
global GLOBAL_OVG_IMPORT
global GLOBAL_OPENVINO_IMPORT
if not GLOBAL_OPENVINO_IMPORT or not GLOBAL_OVG_IMPORT:
if not util.find_spec("openvino") or not util.find_spec("openvino_genai"):
raise LLMWareException(message="OVGenerativeModel: to use OVGenerativeModel requires "
"install of 'openvino' and 'openvino_genai' libraries. "
"Please try: `pip3 install openvino` and "
"`pip3 install openvino_genai` and confirm that your "
"hardware platform is supported.")
if util.find_spec("openvino"):
try:
openvino = importlib.import_module("openvino")
GLOBAL_OPENVINO_IMPORT = True
except:
raise LLMWareException(message="OVGenerativeModel: could not load openvino module.")
if openvino:
if util.find_spec("openvino_genai"):
try:
ovg = importlib.import_module("openvino_genai")
GLOBAL_OVG_IMPORT = True
except:
raise LLMWareException(message="OVGenerativeModel: could not load openvino_genai module.")
if not openvino or not ovg:
raise LLMWareException(message="OVGenerativeModel: could not load required openvino dependencies.")
# end dynamic import here
# set specific parameters associated with custom models
# note - these two parameters will control how prompts are handled - model-specific
self.prompt_wrapper = prompt_wrapper
self.instruction_following = instruction_following
if not model_card:
# safety - empty iterable rather than 'None'
model_card = {}
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
# sets trailing space default when constructing the prompt
# in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves
# performance
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = context_window
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
# key output parameters
self.max_output=max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# eos_token_id set as list to allow for more than one id
self.eos_token_id = []
# use_gpu parameter not used - deprecated
self.use_gpu = False
if "cache_dir" in kwargs:
self.cache_dir = kwargs["cache_dir"]
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load model."
# temperature settings
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.3
self.temperature = 0.3
self.add_prompt_engineering = False
self.add_context = ""
self.context = ""
self.prompt = ""
self.tool_type = ""
self.api_endpoint = api_endpoint
self.pipe = None
self.input_token_count = 0
self.output_token_count = 0
self.params = None
self.model_repo_path = None
self.tokenizer_fn = ""
from llmware.configs import OVConfig
# OVConfig object provided in llmware.configs - in most cases, will not be touched, but
# exposes more options for configuration of the underlying OpenVino implementation
# if config set to CPU - then ensure CPU execution
# note: if set, this will over-ride any other settings
if OVConfig().get_config("device") == "CPU":
self.device = "CPU"
self.optimize_for_gpu_if_available = False
else:
self.optimize_for_gpu_if_available = OVConfig().optimize_for_gpu()
self.generation_version = OVConfig().generation_version()
self.cache = OVConfig().get_config("cache")
self.cache_with_model = OVConfig().get_config("cache_with_model")
self.cache_custom = OVConfig().get_config("cache_custom_path")
self.apply_performance_hints = OVConfig().get_config("apply_performance_hints")
self.use_ov_tokenizer = OVConfig().get_config("use_ov_tokenizer")
self.verbose_mode = OVConfig().get_config("verbose_mode")
self.get_token_counts = OVConfig().get_config("get_token_counts")
# check for llmware path & create if not already set up
if not os.path.exists(LLMWareConfig.get_llmware_path()):
# if not explicitly set up by user, then create folder directory structure
LLMWareConfig.setup_llmware_workspace()
if not os.path.exists(LLMWareConfig.get_model_repo_path()):
os.mkdir(LLMWareConfig.get_model_repo_path())
# please note that the external tokenizer is used solely for producing
# input and output token counts - and can be switched off in OVConfig
if self.get_token_counts:
self.load_ov_external_tokenizer()
self.performance_hints = OVConfig().get_gpu_hints()
self.post_init()
def load_model_for_inference (self, loading_directions,
model_card=None, pipeline=None,**kwargs):
""" Loads OV Model from local path using loading directions. """
global ovg
self.model_repo_path = loading_directions
if model_card:
self.model_card = model_card
if "pipeline" in self.model_card:
self.pipeline = self.model_card["pipeline"]
if pipeline:
self.pipeline = pipeline
self.validate()
if self.device == "GPU" or (self.device == "CPU" and self.optimize_for_gpu_if_available):
device = self.device_resolver()
if device != self.device:
# resets self.device to the resolved device
# if changed, then warning provided by resolver method
self.device = device
if self.device == "GPU" and self.apply_performance_hints:
for k,v in self.performance_hints.items():
try:
# sets GPU performance hints thru openvino core
#TODO: will evaluate if better way to construct/destruct the core object
core = openvino.Core()
core.set_property("GPU", {k:v})
if self.verbose_mode:
logger.info(f"OVGenerativeModel - setting performance hint - {k} - {v}")
except:
logger.warning(f"OVGenerativeModel - unsuccessful setting performance hint - {k} - {v}")
# default is to cache to optimize performance on subsequent loads
# build pipeline based on type
if self.pipeline == "text2image":
self.ov_text_to_image_pipeline()
else:
# default: text2text
self.ov_text_to_text_pipeline()
if self.verbose_mode:
logger.info(f"OVGenerativeModel - completed new pipe creation - "
f"{self.pipeline}")
return self
def device_resolver(self):
""" By default, will look for 'GPU' and if device found, then will select - if no GPU,
then falls back to 'CPU'. """
global ovg
try:
# check if GPU device can be found successfully - if not, auto fallback to CPU device
core = openvino.Core()
gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME")
logger.info(f"OVGenerativeModel - loading - confirmed GPU device name: "
f"{gpu_device_name}")
device = "GPU"
except:
logger.info("OVGenerativeModel - loading - could not find GPU - setting device for CPU")
device = "CPU"
return device
def set_api_key(self, api_key, env_var="USER_MANAGED_OV_API_KEY"):
""" Sets the API key - generally not needed for self-hosted OV models. """
os.environ[env_var] = api_key
logger.info("OVGenerativeModel - added and stored OV api_key in environmental "
"variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_OV_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("OVGenerativeModel - _get_api_key could not successfully "
"retrieve value from: %s ", env_var)
return self.api_key
def load_ov_external_tokenizer(self):
""" Called in class constructor if OVConfig flag set to 'get_output_counts',
and will create a local instance of the tokenizer used to get the counts. """
if "tokenizer_local" in self.model_card:
tok_local_name = self.model_card["tokenizer_local"]
self.tokenizer = LocalTokenizer(tokenizer_fn=tok_local_name)
else:
# if no tokenizer found, then falls back to default tokenizer for 'approximate' count
self.tokenizer = Utilities().get_default_tokenizer()
def ov_text_to_text_pipeline(self):
""" Main entry point for instantiating models """
loading_directions = self.model_repo_path
global ovg
if self.cache:
if self.cache_with_model:
# will put the cache files co-located with the model assets
path_to_cache_dir = loading_directions
else:
path_to_cache_dir = self.cache_custom
if self.verbose_mode:
logger.info(f"OVGenerativeModel - creating pipeline - "
f"{self.device} - {self.cache} - {path_to_cache_dir}")
try:
self.pipe = ovg.LLMPipeline(loading_directions, self.device,
{"CACHE_DIR": path_to_cache_dir})
except:
raise LLMWareException(message=f"OVGenerativeModel - attempt to instantiate LLMPipeline failed - "
f"this could be for a number of reasons, including: "
f"\n1. openvino and openvino_genai installs are not supported "
f"on this os / hardware platform."
f"\n2. the model could not found at path: {loading_directions}, or "
f"\n3. the model may not a valid OpenVino format model.")
else:
#TODO: confirm that empty plugin instructions with no caching will work on all platforms
try:
self.pipe = ovg.LLMPipeline(loading_directions, self.device, {})
except:
raise LLMWareException(message=f"OVGenerativeModel - attempt to instantiate LLMPipeline failed - "
f"this could be for a number of reasons, including: "
f"\n1. openvino and openvino_genai installs are not supported "
f"on this os / hardware platform."
f"\n2. the model could not found at path: {loading_directions}, or "
f"\n3. the model may not a valid OpenVino format model.")
return True
def ov_text_to_image_pipeline(self):
""" Model loading entry point for new OpenVINO text_to_image
pipeline for multimedia models that generate images from text prompt. """
global ovg
# auto set to GPU for faster generation
text_encoder_device = "GPU"
unet_device = "GPU"
vae_decoder_device = "GPU"
width = 512
height = 512
self.pipe = ovg.Text2ImagePipeline(self.model_repo_path)
self.pipe.reshape(1, height, width, self.pipe.get_generation_config().guidance_scale)
properties = {"CACHE_DIR": self.model_repo_path}
self.pipe.compile(text_encoder_device, unet_device, vae_decoder_device, config=properties)
return True
def text_to_image_gen(self, prompt, image_name):
""" Specialized generation function for image generating models. """
from PIL import Image
# experiment with different step numbers
# will expose as parameter in future releases
number_of_inference_steps_per_image = 10
tmp_path = LLMWareConfig().get_tmp_path()
img_path = os.path.join(tmp_path, str(image_name) + ".bmp")
image_tensor = self.pipe.generate(prompt,
num_inference_steps=number_of_inference_steps_per_image)
image = Image.fromarray(image_tensor.data[0])
image.save(img_path)
return img_path
def ov_token_counter(self, text):
""" Called twice in inference generation loop to get the input_token_count and
output_token_count. This step can be skipped by setting the OVConfig as follows:
`from llmware.configs import OVConfig
OVConfig().set_config("get_token_counts", False)`
In our testing, the performance impact is negligible, but may be different in your
environment and use case.
If this is set to False, then no token counts will be provided in the usage totals.
"""
if self.tokenizer:
toks = len(self.tokenizer.encode(text))
else:
toks = 0
return toks
def prompt_engineer(self, query, context, inference_dict):
""" Applies prompt and templating preparation. """
# if loaded model was not pretrained on instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper,
instruction=None)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def _generate_ov_genai(self, prompt, streamer=None):
""" Core generation script provided by generation loop exposed in the OpenVino_GenAI library. """
global ovg
if self.verbose_mode:
logger.info("OVGenerativeModel - calling openvino_genai backend in _generate_ov_genai method.")
config = ovg.GenerationConfig()
config.max_new_tokens = self.max_output
# prevent error in generation if sampling True and temperature is set to 0.0
if self.sample and self.temperature == 0.0:
self.temperature = 0.2
logger.warning(f"OVGenerativeModel - since sample is set to True, adjusting "
f"temperature from 0.0 to small value - 0.2 - to avoid error "
f"in the generation loop.")
config.temperature = self.temperature
config.do_sample = self.sample
# core generation step - runs generation loop on pipe with prompt and config
if streamer:
output = self.pipe.generate(prompt, config, streamer=streamer)
else:
output = self.pipe.generate(prompt, config)
return output
@staticmethod
def ov_default_streamer(x):
""" Stream to console - used by default in stream method -
can be over-ridden by passing a custom streaming function to
the stream generate call. """
print(x, end="", flush=True)
return ovg.StreamingStatus.RUNNING
def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None):
""" Executes generation inference on model. """
# first prepare the prompt
self.prompt = prompt
if add_context:
self.add_context = add_context
self.context = self.add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
# show warning if function calling model
if self.fc_supported:
logger.warning("OVGenerativeModel - this is a function calling model - using .inference may lead "
"to unexpected results. Recommended to use the .function_call method to ensure "
"correct prompt template packaging.")
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# text_prompt = prompt_final + "\n"
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
text_prompt = prompt_final + self.trailing_space
# counts the input tokens
if self.get_token_counts:
self.input_token_count = self.ov_token_counter(text_prompt)
else:
self.input_token_count = 0
time_start = time.time()
# main call to inner generate function
output = self._generate_ov_genai(text_prompt)
output_str = output
# post-processing clean-up - stop at endoftext
eot = output_str.find("<|endoftext|>")
if eot > -1:
output_str = output_str[:eot]
# new post-processing clean-up - stop at </s>
eots = output_str.find("</s>")
if eots > -1:
output_str = output_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = output_str.find("<bot>:")
if bot > -1:
output_str = output_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = output_str.find("<s>")
if boss > -1:
output_str = output_str[boss + len("<s>"):]
# end - post-processing
# counts the output tokens
if self.get_token_counts:
self.output_token_count = self.ov_token_counter(output_str)
else:
self.output_token_count = 0
usage = {"input": self.input_token_count,
"output": self.output_token_count,
"total": self.input_token_count + self.output_token_count,
"metric": "tokens",
"processing_time": time.time() - time_start}
output_response = {"llm_response": output_str, "usage": usage}
self.get_logits = False
# output inference parameters
self.llm_response = output_str
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def fc_prompt_engineer(self, context, params=None, function=None):
""" Prompt engineering for Function Call prompts. """
if not params:
params = self.primary_keys
if not function:
function = self.function[0]
# prepare SLIM prompt
class_str = ""
for key in params:
class_str += str(key) + ", "
if class_str.endswith(", "):
class_str = class_str[:-2]
f = str(function)
# key templating format for SLIM function calls
full_prompt = "<human>: " + context + "\n" + "<{}> {} </{}>".format(f, class_str, f) + "\n<bot>:"
full_prompt = full_prompt + self.trailing_space
return full_prompt
def function_call(self, context, function=None, params=None, get_logits=False,
temperature=-99, max_output=None):
""" This is the key inference method for SLIM models - takes a context passage and a key list
which is packaged in the prompt as the keys for the dictionary output"""
self.context = context
if not self.fc_supported:
logger.warning("OVGenerativeModel - loaded model does not support function calls. "
"Please either use the standard .inference method with this model, or use a "
"model that has 'function_calls' key set to True in its model card.")
return []
# reset and start from scratch with new function call
self.output_tokens = []
self.logits_record = []
if temperature != -99:
self.temperature = temperature
if max_output:
self.target_requested_output_tokens = max_output
if get_logits:
logger.warning("OVGenerativeModel - current implementation does not support get_logits option.")
self.get_logits = False
if params:
self.primary_keys = params
if function:
self.function = function
if not self.primary_keys:
logger.warning("OVGenerativeModel - function call - no keys provided - function call may "
"yield unpredictable results")
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.function_call_over_api_endpoint(model_name=self.model_name,
context=self.context,params=self.primary_keys,
function=self.function,
api_key=self.api_key,get_logits=self.get_logits)
# END - route to api endpoint
prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=function)
time_start = time.time()
# counts the input tokens
if self.get_token_counts:
self.input_token_count = self.ov_token_counter(prompt)
else:
self.input_token_count = 0
# main call to inner generate function
output_str = self._generate_ov_genai(prompt)
# post-processing clean-up - stop at endoftext
eot = output_str.find("<|endoftext|>")
if eot > -1:
output_str = output_str[:eot]
# new post-processing clean-up - stop at </s>
eots = output_str.find("</s>")
if eots > -1:
output_str = output_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = output_str.find("<bot>:")
if bot > -1:
output_str = output_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = output_str.find("<s>")
if boss > -1:
output_str = output_str[boss + len("<s>"):]
# end - post-processing
# counts the output tokens
if self.get_token_counts:
self.output_token_count = self.ov_token_counter(output_str)
else:
self.output_token_count = 0
usage = {"input": self.input_token_count,
"output": self.output_token_count,
"total": self.input_token_count + self.output_token_count,
"metric": "tokens",
"processing_time": time.time() - time_start}
try:
output_value = ast.literal_eval(output_str)
output_type = "dict"
# allow for multiple valid object types - will expand over time
if isinstance(output_value,dict): output_type = "dict"
if isinstance(output_value,list): output_type = "list"
usage.update({"type": output_type})
except:
# could not convert automatically to python object
output_type = "string"
usage.update({"type": output_type})
output_value = output_str
# auto remediate set to False - turning off this capability currently
self.auto_remediate_function_call_output = False
if self.auto_remediate_function_call_output:
# attempt to remediate
output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str)
usage.update({"type": output_type, "remediation": True})
output_value = output_rem
if output_type == "string":
logger.warning("OVGenerativeModel - automatic conversion of function call output failed, "
"and attempt to remediate was not successful - %s ", output_str)
else:
logger.info("OVGenerativeModel - function call output could not be automatically "
"converted, but remediation was successful to type - %s ", output_type)
output_response = {"llm_response": output_value, "usage": usage}
# get_logits - not currently implemented
if get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output_value
self.usage = usage
self.final_prompt = prompt
self.register()
return output_response
def unload_model(self):
""" Resetting the pipe removes pointer to pipeline in Python, and generally triggers a (safe) release of
the memory. WIP - will continue to evaluate effectiveness across use patterns and platforms. """
self.pipe = None
return True
def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False):
""" Called by .inference method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
import ast
import requests
self.prompt = prompt
self.context = context
url = self.api_endpoint + "{}".format("/")
output_raw = requests.post(url, data={"model_name": self.model_name,
"question": self.prompt,
"context": self.context,
"api_key": self.api_key,
"max_output": self.max_output,
"temperature": self.temperature})
try:
output = json.loads(output_raw.text)
# will attempt to unpack logits - but catch any exceptions and skip
if "logits" in output:
try:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
except:
output["logits"] = []
# will attempt to unpack output tokens - but catch any exceptions and skip
if "output_tokens" in output:
try:
# ot_int = [int(x) for x in output["output_tokens"]]
# output["output_tokens"] = ot_int
output_tokens = ast.literal_eval(output["output_tokens"])
output["output_tokens"] = output_tokens
except:
output["output_tokens"] = []
except:
logger.warning("OVGenerativeModel - api inference was not successful")
output = {"llm_response": "api-inference-error", "usage": {}}
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None, streamer=None):
""" Executes stream generation inference on model.
NOTE: operates differently than other stream methods in LLMWare -
the method is not a generator, but rather the streaming update is
provided through passing a streamer function to the OpenVINO
backend - which will be called at each step of the generation
cycle.
Sample call:
# will automatically use default streamer to print to console
response = model.stream('Where is Paris?')
# pass a custom streaming function
response = model.stream('Where is Rome?', streamer=my_streamer)
Streamer function example: .ov_default_streamer in this model class
"""
# first prepare the prompt
self.prompt = prompt
if add_context:
self.add_context = add_context
self.context = self.add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
# show warning if function calling model
if self.fc_supported:
logger.warning("OVGenerativeModel - this is a function calling model - using .inference may lead "
"to unexpected results. Recommended to use the .function_call method to ensure "
"correct prompt template packaging.")
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.trailing_space
# counts the input tokens
if self.get_token_counts:
self.input_token_count = self.ov_token_counter(text_prompt)
else:
self.input_token_count = 0
time_start = time.time()
# main call to inner generate function
if not streamer:
streamer = self.ov_default_streamer
output = self._generate_ov_genai(text_prompt, streamer=streamer)
output_str = output
# post-processing clean-up - stop at endoftext
eot = output_str.find("<|endoftext|>")
if eot > -1:
output_str = output_str[:eot]
# new post-processing clean-up - stop at </s>
eots = output_str.find("</s>")
if eots > -1:
output_str = output_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = output_str.find("<bot>:")
if bot > -1:
output_str = output_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = output_str.find("<s>")
if boss > -1:
output_str = output_str[boss + len("<s>"):]
# end - post-processing
# counts the output tokens
if self.get_token_counts:
self.output_token_count = self.ov_token_counter(output_str)
else:
self.output_token_count = 0
usage = {"input": self.input_token_count,
"output": self.output_token_count,
"total": self.input_token_count + self.output_token_count,
"metric": "tokens",
"processing_time": time.time() - time_start}
output_response = {"llm_response": output_str, "usage": usage}
self.get_logits = False
# output inference parameters
self.llm_response = output_str
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="",
function=None, endpoint_base=None, api_key=None, get_logits=False):
""" Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
# send to api agent server
self.context = context
self.tool_type = tool_type
self.prompt = prompt
import ast
import requests
if endpoint_base:
self.api_endpoint = endpoint_base
if api_key:
# e.g., "demo-test"
self.api_key = api_key
if not params:
self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type]
mc = ModelCatalog().lookup_model_card(self.model_name)
if "primary_keys" in mc:
params = mc["primary_keys"]
self.primary_keys = params
if function:
self.function = function
self.context = context
self.preview()
url = self.api_endpoint + "{}".format("/agent")
output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key,
"tool_type": self.tool_type,
"function": self.function, "params": self.primary_keys, "max_output": 50,
"temperature": 0.0, "sample": False, "prompt": self.prompt,
"context": self.context, "get_logits": True})
try:
# output = ast.literal_eval(output_raw.text)
output = json.loads(output_raw.text)
if "logits" in output:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
if "output_tokens" in output:
ot_int = [int(x) for x in output["output_tokens"]]
output["output_tokens"] = ot_int
except:
logger.warning("OVGenerativeModel - api inference was not successful")
output = {}
logger.info(f"OVGenerativeModel - executed Agent call over API endpoint - "
f"{model_name} - {function} - {output}")
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
class OVVisionGenerativeModel(BaseModel):
""" OVVisionGenerativeModel class implements the OpenVino generative model interface for fast inference
performance on x86 Intel architectures, including both Intel CPU and GPU. """
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
prompt_wrapper=None, instruction_following=False, context_window=2048,
sample=False,max_output=100, temperature=0.0,
get_logits=False, api_endpoint=None, device="GPU",
pipeline="image2text", **kwargs):
super().__init__()
self.model_class = "OVVisionGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = model
self.tokenizer = tokenizer
self.sample=sample
self.get_logits=get_logits
self.pipeline = pipeline
if get_logits:
logger.warning(f"OVGenerativeModel - current implementation does not support "
f"get_logits option.")
self.get_logits = False
self.auto_remediate_function_call_output = True
# Function Call parameters
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
self.cache_dir = None
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
# will look for special cache_dir set in the model card
# can be over-ridden if passed as kwarg in loading model
if "cache_dir" in model_card:
self.cache_dir = model_card["cache_dir"]
if "pipeline" in model_card:
self.pipeline = model_card["pipeline"]
# insert dynamic openvino load here
if not api_endpoint:
global openvino
global ovg
global GLOBAL_OVG_IMPORT
global GLOBAL_OPENVINO_IMPORT
if not GLOBAL_OPENVINO_IMPORT or not GLOBAL_OVG_IMPORT:
if not util.find_spec("openvino") or not util.find_spec("openvino_genai"):
raise LLMWareException(message="OVGenerativeModel: to use OVGenerativeModel requires "
"install of 'openvino' and 'openvino_genai' libraries. "
"Please try: `pip3 install openvino` and "
"`pip3 install openvino_genai` and confirm that your "
"hardware platform is supported.")
if util.find_spec("openvino"):
try:
openvino = importlib.import_module("openvino")
GLOBAL_OPENVINO_IMPORT = True
except:
raise LLMWareException(message="OVGenerativeModel: could not load openvino module.")
if openvino:
if util.find_spec("openvino_genai"):
try:
ovg = importlib.import_module("openvino_genai")
GLOBAL_OVG_IMPORT = True
except:
raise LLMWareException(message="OVGenerativeModel: could not load openvino_genai module.")
if not openvino or not ovg:
raise LLMWareException(message="OVGenerativeModel: could not load required openvino dependencies.")
# end dynamic import here
# set specific parameters associated with custom models
# note - these two parameters will control how prompts are handled - model-specific
self.prompt_wrapper = prompt_wrapper
self.instruction_following = instruction_following
if not model_card:
# safety - empty iterable rather than 'None'
model_card = {}
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
# sets trailing space default when constructing the prompt
# in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves
# performance
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = context_window
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
# key output parameters
self.max_output=max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# eos_token_id set as list to allow for more than one id
self.eos_token_id = []
# use_gpu parameter not used - deprecated
self.use_gpu = False
self.device = device
if "device" in kwargs:
self.device = kwargs["device"]
if "cache_dir" in kwargs:
self.cache_dir = kwargs["cache_dir"]
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load model."
# temperature settings
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.3
self.temperature = 0.3
self.add_prompt_engineering = False
self.add_context = ""
self.context = ""
self.prompt = ""
self.tool_type = ""
self.api_endpoint = api_endpoint
self.pipe = None
self.input_token_count = 0
self.output_token_count = 0
self.params = None
self.model_repo_path = None
self.tokenizer_fn = ""
from llmware.configs import OVConfig
# OVConfig object provided in llmware.configs - in most cases, will not be touched, but
# exposes more options for configuration of the underlying OpenVino implementation
# if config set to CPU - then ensure CPU execution
if OVConfig().get_config("device") == "CPU":
self.device = "CPU"
self.optimize_for_gpu_if_available = False
else:
self.optimize_for_gpu_if_available = OVConfig().optimize_for_gpu()
self.generation_version = OVConfig().generation_version()
self.cache = OVConfig().get_config("cache")
self.cache_with_model = OVConfig().get_config("cache_with_model")
self.cache_custom = OVConfig().get_config("cache_custom_path")
self.apply_performance_hints = OVConfig().get_config("apply_performance_hints")
self.use_ov_tokenizer = OVConfig().get_config("use_ov_tokenizer")
self.verbose_mode = OVConfig().get_config("verbose_mode")
self.get_token_counts = OVConfig().get_config("get_token_counts")
# check for llmware path & create if not already set up
if not os.path.exists(LLMWareConfig.get_llmware_path()):
# if not explicitly set up by user, then create folder directory structure
LLMWareConfig.setup_llmware_workspace()
if not os.path.exists(LLMWareConfig.get_model_repo_path()):
os.mkdir(LLMWareConfig.get_model_repo_path())
# please note that the external tokenizer is used solely for producing
# input and output token counts - and can be switched off in OVConfig
if self.get_token_counts:
self.load_ov_external_tokenizer()
self.performance_hints = OVConfig().get_gpu_hints()
self.post_init()
def load_model_for_inference (self, loading_directions,
model_card=None, pipeline=None,**kwargs):
""" Loads OV Model from local path using loading directions. """
global ovg
self.model_repo_path = loading_directions
if model_card:
self.model_card = model_card
self.validate()
if self.device == "GPU" or (self.device == "CPU" and self.optimize_for_gpu_if_available):
device = self.device_resolver()
if device != self.device:
# resets self.device to the resolved device
# if changed, then warning provided by resolver method
self.device = device
if self.device == "GPU" and self.apply_performance_hints:
for k,v in self.performance_hints.items():
try:
# sets GPU performance hints thru openvino core
core = openvino.Core()
core.set_property("GPU", {k:v})
if self.verbose_mode:
logger.info(f"OVVisionGenerativeModel - setting performance hint - {k} - {v}")
except:
logger.warning(f"OVVisionGenerativeModel - unsuccessful setting performance hint - {k} - {v}")
# default is to cache to optimize performance on subsequent loads
properties = {"CACHE_DIR": self.model_repo_path}
self.pipe = ovg.VLMPipeline(self.model_repo_path, self.device,**properties)
if self.verbose_mode:
logger.info(f"OVVisionGenerativeModel - completed new pipe creation - "
f"{self.model_name} - on device {self.device}")
return self
def device_resolver(self):
""" By default, will look for 'GPU' and if device found, then will select - if no GPU,
then falls back to 'CPU'. """
global ovg
try:
# check if GPU device can be found successfully - if not, auto fallback to CPU device
core = openvino.Core()
gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME")
logger.info(f"OVVisionGenerativeModel - loading - confirmed GPU device name: "
f"{gpu_device_name}")
device = "GPU"
except:
logger.info("OVVisionGenerativeModel - loading - could not find GPU - setting device for CPU")
device = "CPU"
return device
def load_ov_external_tokenizer(self):
""" Called in class constructor if OVConfig flag set to 'get_output_counts',
and will create a local instance of the tokenizer used to get the counts. """
if "tokenizer_local" in self.model_card:
tok_local_name = self.model_card["tokenizer_local"]
self.tokenizer = LocalTokenizer(tokenizer_fn=tok_local_name)
else:
# if no tokenizer found, then falls back to default tokenizer for 'approximate' count
self.tokenizer = Utilities().get_default_tokenizer()
def inference(self, prompt, image_path, inference_dict=None):
""" Implemented as stream without a streamer function. """
return self.stream(prompt,image_path, inference_dict=inference_dict,
streamer=None, no_stream=True)
def stream(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None, streamer=None,no_stream=False):
""" Executes stream generation inference on model.
NOTE: operates differently than other stream methods in LLMWare -
the method is not a generator, but rather the streaming update is
provided through passing a streamer function to the OpenVINO
backend - which will be called at each step of the generation
cycle.
Sample call:
# will automatically use default streamer to print to console
response = model.stream('Describe this image', 'C:\\Users\\...')
# pass a custom streaming function
response = model.stream('Describe this image' 'C:\\Users\\...', streamer=my_streamer)
Streamer function example: .ov_default_streamer in this model class
"""
# first prepare the prompt
self.prompt = prompt
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
text_prompt = self.prompt
# counts the input tokens
if self.get_token_counts:
self.input_token_count = self.ov_token_counter(text_prompt)
else:
self.input_token_count = 0
time_start = time.time()
# prepares the image as tensor
from PIL import Image
pic = Image.open(image_path).convert("RGB")
image_data = np.array(pic)[None]
images = [openvino.Tensor(image_data)]
# main call to inner generate function
if not streamer and not no_stream:
streamer = self.ov_default_streamer
output = self._generate_ov_genai(text_prompt,
image=images,
streamer=streamer)
output_str = output
self.output_token_count = 0
usage = {"input": self.input_token_count,
"output": self.output_token_count,
"total": self.input_token_count + self.output_token_count,
"metric": "tokens",
"processing_time": time.time() - time_start}
output_response = {"llm_response": output_str, "usage": usage}
self.get_logits = False
# output inference parameters
self.llm_response = output_str
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def ov_token_counter(self, text):
""" Called twice in inference generation loop to get the input_token_count and
output_token_count. This step can be skipped by setting the OVConfig as follows:
`from llmware.configs import OVConfig
OVConfig().set_config("get_token_counts", False)`
In our testing, the performance impact is negligible, but may be different in your
environment and use case.
If this is set to False, then no token counts will be provided in the usage totals.
"""
if self.tokenizer:
toks = len(self.tokenizer.encode(text))
else:
toks = 0
return toks
def prompt_engineer(self, query, context, inference_dict):
""" Implemented by openvino_genai module """
pass
def _generate_ov_genai(self, prompt, image=None, streamer=None):
""" Core generation script provided by generation loop exposed in the OpenVino_GenAI library. """
global ovg
config = ovg.GenerationConfig()
config.max_new_tokens = self.max_output
self.sample=False
self.temperature =0.0
# prevent error in generation if sampling True and temperature is set to 0.0
if self.sample and self.temperature == 0.0:
self.temperature = 0.2
logger.warning(f"OVVisionGenerativeModel - since sample is set to True, adjusting "
f"temperature from 0.0 to small value - 0.2 - to avoid error "
f"in the generation loop.")
config.temperature = self.temperature
config.do_sample = self.sample
logger.info("OVVisionGenerativeModel - _generate_ov_genai - "
f"do_sample is {self.sample} with temperature - {self.temperature}")
# core generation step - runs generation loop on pipe with prompt and config
if image:
output = self.pipe.generate(prompt,image,config, streamer=streamer)
else:
if streamer:
output = self.pipe.generate(prompt, config, streamer=streamer)
else:
output = self.pipe.generate(prompt, config)
# need to unpack the output
text_output = ""
if output:
if hasattr(output, "texts"):
text_output = output.texts
return text_output
@staticmethod
def ov_default_streamer(x):
""" Stream to console - used by default in stream method -
can be over-ridden by passing a custom streaming function to
the stream generate call. """
print(x, end="", flush=True)
return ovg.StreamingStatus.RUNNING
def unload_model(self):
""" Resetting the pipe removes pointer to pipeline in Python, and generally triggers a (safe) release of
the memory. WIP - will continue to evaluate effectiveness across use patterns and platforms. """
self.pipe = None
return True
class OpenChatModel(BaseModel):
""" OpenChatModel class implements the OpenAI prompt API and is intended for use with OpenChat compatible
inference servers """
def __init__(self, model_name=None, model_card=None, context_window=4000,prompt_wrapper=None, api_key="not_used",
**kwargs):
super().__init__(**kwargs)
# expected to take config parameters from model card
self.api_key = api_key
self.model_name = model_name
self.model_card = model_card
self.model_class = "OpenChatModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
# by default, will use the 'chat' open interface, but alternative is 'completion' api
self.model_type = "chat"
# assume that prompt_wrapper is set in the model card configuration
self.prompt_wrapper = prompt_wrapper
# this is the key parameter that needs to be configured to pass to open chat inference server
self.api_base = ""
if self.model_card:
if "model_type" in self.model_card:
self.model_type = self.model_card["model_type"]
if "api_base" in self.model_card:
self.api_base = self.model_card["api_base"]
if "prompt_wrapper" in self.model_card:
self.prompt_wrapper = self.model_card["prompt_wrapper"]
self.error_message = "\nUnable to connect to OpenChat Model. Please try again later."
self.separator = "\n"
# assume input (50%) + output (50%)
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings
self.temperature = 0.7
self.target_requested_output_tokens = 100
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
# new post_init check
self.post_init()
def set_api_key (self, api_key, env_var="USER_MANAGED_OPEN_CHAT_API_KEY"):
""" Utility method to set API key if needed. """
# set api_key
os.environ[env_var] = api_key
logger.info("update: added and stored OpenChat api_key in environmental variable- %s", env_var)
return self
def _get_api_key (self, env_var="USER_MANAGED_OPEN_CHAT_API_KEY"):
""" Utility method to get API key if needed. """
# not expected to use api_key - so may be empty - handled in inference separately
self.api_key = os.environ.get(env_var)
return self.api_key
def token_counter(self, text_sample):
""" Gets GPT2 tokenizer for fast approximate token counting. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer_chat(self, query, context, inference_dict=None):
""" Creates Prompt Template for Chat Interaction. """
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
system_message = prompt_dict["prompt_card"]["system_message"]
if not system_message:
system_message = "You are a helpful assistant."
core_prompt = prompt_dict["core_prompt"]
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper, instruction=None)
messages = [
{"role": "system", "content": system_message},
{"role": "user", "content": core_prompt}
]
return messages
def prompt_engineer_completion (self, query, context, inference_dict=None):
""" Creates Prompt for 'Completion' style interface. """
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
core_prompt = prompt_dict["core_prompt"]
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper, instruction=None)
return core_prompt
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on the Model. Required input is a text prompt. Optional parameters include
an 'add_context' to be used as a source in the prompt, and assembled according to the prompt
engineering style (e.g., add_prompt_engineering). An optional inference_dict can include other optional
parameters such as temperature and max_tokens. If an API key is required, it can be passed here, or
will be picked up through the appropriate os.environ variable """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# api_key
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
# call to preview (not implemented by default)
self.preview()
# expect that .api_base will route to local open chat inference server
# -- assumed that *** api_key likely not used ***
# -- in openai >= 1.0: .api_base replaced with 'base_url' attribute
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
if not self.api_key:
client = OpenAI(api_key="not-used",base_url=self.api_base)
else:
client = OpenAI(api_key=self.api_key,base_url=self.api_base)
# default case - pass the prompt received without change
prompt_enriched = self.prompt
usage = {}
time_start = time.time()
try:
if self.model_type == "chat":
messages = self.prompt_engineer_chat(prompt_enriched, self.add_context, inference_dict)
# using openai >1.0 api -> create client object, and output is pydantic, not dicts
response = client.chat.completions.create(model=self.model_name,messages=messages,
max_tokens=self.target_requested_output_tokens)
""" assume 'minimal' api output conformance with OpenAI """
text_out = response.choices[0].message.content
""" note: some openchat api do not support providing usage output consistent with OpenAI API """
pt = 0
ct = 0
tt = 0
""" best effort to gather usage data if conforms with OpenAI """
if hasattr(response, "usage"):
if hasattr(response.usage, "prompt_tokens"):
pt = response.usage.prompt_tokens
if hasattr(response.usage, "completion_tokens"):
ct = response.usage.completion_tokens
if hasattr(response.usage, "total_tokens"):
tt = response.usage.total_tokens
usage = {"input": pt,
"output": ct,
"total": tt,
"metric": "tokens",
"processing_time": time.time() - time_start}
else:
# traditional completion 'instruct gpt' models
prompt_enriched = self.prompt_engineer_completion(prompt_enriched,
self.add_context,
inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.separator
response = client.completions.create(model=self.model_name, prompt=text_prompt,
temperature=self.temperature,
max_tokens=self.target_requested_output_tokens)
""" assume 'minimal' api output conformance with OpenAI """
text_out = response.choices[0].text
""" note: some openchat api do not support providing usage output consistent with OpenAI API """
pt = 0
ct = 0
tt = 0
""" best effort to gather usage data if conforms with OpenAI API """
if hasattr(response, "usage"):
if hasattr(response.usage, "prompt_tokens"):
pt = response.usage.prompt_tokens
if hasattr(response.usage, "completion_tokens"):
ct = response.usage.completion_tokens
if hasattr(response.usage, "total_tokens"):
tt = response.usage.total_tokens
usage = {"input": pt,
"output": ct,
"total": tt,
"metric": "tokens",
"processing_time": time.time() - time_start}
except Exception as e:
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"Open Chat model inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class OllamaModel(BaseModel):
""" OllamaModel class implements the Ollama model prompt API and is intended for use in building
RAG pipelines while using a Ollama endpoint primarily for rapid local prototyping. """
def __init__(self, model_name=None, model_card=None, context_window=4000,prompt_wrapper=None, api_key="not_used",
**kwargs):
super().__init__(**kwargs)
self.model_class = "OllamaModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
# default ollama specific settings
# self.uri = "http://localhost:11434/api/"
self.host = "localhost"
self.port = 11434
self.model_name = "llama2"
self.model_type = "chat"
self.stream_mode = False
self.raw_mode = False
# expected to take config parameters from model card
self.api_key = api_key
self.model_name = model_name
self.model_card = model_card
# assume that prompt_wrapper is set in the model card configuration
self.prompt_wrapper = prompt_wrapper
if self.model_card:
if "model_name" in self.model_card:
self.model_name = self.model_card["model_name"]
if "model_type" in self.model_card:
self.model_type = self.model_card["model_type"]
if "host" in self.model_card:
self.host = self.model_card["host"]
if "port" in self.model_card:
self.port = self.model_card["port"]
if "prompt_wrapper" in self.model_card:
self.prompt_wrapper = self.model_card["prompt_wrapper"]
if "raw_mode" in self.model_card:
self.raw_mode = self.model_card["raw_mode"]
if "stream_mode" in self.model_card:
self.stream_mode = self.model_card["stream_mode"]
self.error_message = f"\nUnable to connect to Ollama Model. Please check that Ollama is running"\
f"at {self.host}:{self.port}"
self.separator = "\n"
# assume input (50%) + output (50%)
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings -> not used as generation handled by Ollama inference
self.temperature = 0.7
self.target_requested_output_tokens = 100
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
# self.uri = "http://localhost:11434/api/"
self.uri = f"http://{self.host}:{self.port}/api/"
self.post_init()
def set_api_key (self, api_key, env_var="USER_MANAGED_OLLAMA_API_KEY"):
""" Utility method to store api_key in os.environ variable. """
# set api_key
os.environ[env_var] = api_key
logger.info("update: added and stored Ollama api_key in environmental variable- %s", env_var)
return self
def _get_api_key (self, env_var="USER_MANAGED_OLLAMA_API_KEY"):
""" Utility method to get api_key from os.environ variable. """
self.api_key = os.environ.get(env_var)
return self.api_key
def token_counter(self, text_sample):
""" Uses default GPT2 tokenizer for fast, approximate token count, if needed. """
# note: this is an approximation for counting the input tokens using a default tokenizer
# --to get 100% accurate, need to use the tokenizer being applied on the 'ollama' decoding
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer (self, query, context, inference_dict=None):
""" Builds prompt by assembling query, context and applying the selected prompt style. """
# by default, this will construct a very basic prompt, concatenating the
# query + context with a basic instruction
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
core_prompt = prompt_dict["core_prompt"]
# Ollama will handle the prompt wrap templating, unless self.raw_mode = True
if self.raw_mode:
if self.prompt_wrapper:
core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper,
instruction=None)
return core_prompt
def discover_models(self):
""" Calls Ollama endpoint for discovery of available models and their locations. """
response = requests.get(self.uri+"tags")
logger.info("update: OllamaModel - discover_models - %s ", response.text)
output = json.loads(response.text)
return output
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" In typical case with raw_mode = False, then no prompt engineering, just apply a basic
assembly of the prompt and context. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# call to preview hook (not implemented by default)
self.preview()
# default case - pass the prompt received without change
prompt_enriched = self.prompt
usage = {}
time_start = time.time()
try:
# assumes 'chat' api by default
if self.model_type == "chat":
full_prompt = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict)
messages = [{"role": "user", "content": full_prompt}]
uri = self.uri + "chat"
response = requests.post(uri,
json={"model": self.model_name,
"messages": messages, "stream": self.stream_mode})
logger.info("update: OllamaModel response - chat - %s ", response.text)
output = json.loads(response.text)
text_out = output["message"]["content"]
pt = 0
ct = 0
tt = 0
""" best effort to gather usage data """
if "eval_count" in output:
ct = output["eval_count"]
tt += ct
pt = self.token_counter(full_prompt)
tt += pt
usage = {"input": pt,
"output": ct,
"total": tt,
"metric": "tokens",
"processing_time": time.time() - time_start}
else:
# traditional completion 'instruct gpt' api
prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context,
inference_dict=inference_dict)
prompt_final = prompt_enriched + self.separator
params = {"model": self.model_name, "prompt": prompt_final, "stream": self.stream_mode}
# response = requests.post("http://localhost:11434/api/generate", json=params)
response = requests.post(self.uri+"generate", json=params)
output = json.loads(response.text)
text_out = output["response"]
pt = 0
ct = 0
tt = 0
""" best effort to gather usage data if conforms with OpenAI API """
if "eval_count" in output:
ct = output["eval_count"]
tt += ct
pt = self.token_counter(prompt_final)
tt += pt
usage = {"input": pt,
"output": ct,
"total": tt,
"metric": "tokens",
"processing_time": time.time() - time_start}
except Exception as e:
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"error: Ollama model inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class OpenAIGenModel(BaseModel):
""" OpenAIGenModel class implements the OpenAI API for its generative decoder models. """
def __init__(self, model_name=None, api_key=None, context_window=32768,
max_output=1000,temperature=0.0, **kwargs):
super().__init__(**kwargs)
self.model_class = "OpenAIGenModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.api_key = api_key
self.model_name = model_name
self.error_message = "\nUnable to connect to OpenAI. Please try again later."
self.separator = "\n"
# assume input (50%) + output (50%)
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings
if temperature >= 0.0:
self.temperature = temperature
else:
self.temperature = 0.0
self.target_requested_output_tokens = max_output
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
self.context = ""
# provides option to pass custom openai_client to model class at inference time
self.openai_client = None
if "model_card" in kwargs:
self.model_card = kwargs["model_card"]
else:
self.model_card = {}
self.post_init()
def set_api_key (self, api_key, env_var="OPENAI_API_KEY"):
""" Utility method to set the API key in os.environ variable. """
# set api_key
os.environ[env_var] = api_key
logger.info(f"OpenAIGenModel - added and stored OpenAI api_key in environmental variable- {env_var}")
return self
def _get_api_key (self, env_var="OPENAI_API_KEY"):
""" Utility method to get the API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error(f"OpenAIGenModel - _get_api_key could not successfully retrieve "
f"value from: {env_var}")
return self.api_key
def token_counter(self, text_sample):
""" Fast, approximate token counting using GPT2 tokenizer. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer_chatgpt3(self, query, context, inference_dict=None):
""" Builds prompt in ChatGPT format. """
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
system_message = prompt_dict["prompt_card"]["system_message"]
if not system_message:
system_message = "You are a helpful assistant."
core_prompt = prompt_dict["core_prompt"]
messages = [
{"role": "system", "content": system_message},
{"role": "user", "content": core_prompt}
]
return messages
def prompt_engineer (self, query, context, inference_dict=None):
""" Builds Prompt in traditional 'completion' style. """
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
core_prompt = prompt_dict["core_prompt"]
return core_prompt
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on OpenAI Model. Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if "openai_client" in inference_dict:
self.openai_client = inference_dict["openai_client"]
from llmware.configs import OpenAIConfig
if not self.openai_client:
azure_client = OpenAIConfig().get_azure_client()
else:
azure_client = self.openai_client
# api_key
if api_key:
self.api_key = api_key
if not self.api_key:
if not azure_client:
self.api_key = self._get_api_key()
if not self.api_key and not azure_client:
raise LLMWareException(message="OpenAIGenModel: no api_key found for OpenAI. This can be set as "
"an environment variable with: os.environ['OPENAI_API_KEY'] = '...'")
# call to preview hook (not implemented by default)
self.preview()
# default case - pass the prompt received without change
prompt_enriched = self.prompt
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
usage = {}
time_start = time.time()
try:
if self.model_name in ["gpt-4o", "o4-mini"]:
# PATH #1 - the new 'responses' endpoint
messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict)
# updated OpenAI client to >v1.0 API - create client, and returns pydantic objects
if not azure_client:
client = OpenAI(api_key=self.api_key)
model_name = self.model_name
else:
logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig")
client = azure_client
# adapt model name for azure, e.g., replace(".", "")
model_name = OpenAIConfig().get_azure_model_name(self.model_name)
response = client.responses.create(model=model_name,input=messages,)
text_out = response.output_text
usage = {"input": response.usage.input_tokens,
"output": response.usage.output_tokens,
"total": response.usage.total_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
elif self.model_name in ["gpt-5.2-pro", "gpt-5.2", "gpt-5-mini", "gpt-5-nano", "gpt-4.1"]:
# PATH #2 - 'main' chatgpt-style chat completions endpoint
messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict)
# updated OpenAI client to >v1.0 API - create client, and returns pydantic objects
if not azure_client:
client = OpenAI(api_key=self.api_key)
model_name = self.model_name
else:
logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig")
client = azure_client
# adapt model name for azure, e.g., replace(".", "")
model_name = OpenAIConfig().get_azure_model_name(self.model_name)
# note: max_tokens deprecated for max_output_tokens -> but not supported for 'o' models
response = client.chat.completions.create(model=model_name, messages=messages)
text_out = response.choices[0].message.content
usage = {"input": response.usage.prompt_tokens,
"output": response.usage.completion_tokens,
"total": response.usage.total_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
else:
# PATH #3 - openai traditional 'instruct gpt' completion models
prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.separator
azure_client = OpenAIConfig().get_azure_client()
if not azure_client:
client = OpenAI(api_key=self.api_key)
model_name = self.model_name
else:
logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig")
client = azure_client
# adapt model name for azure, e.g., replace(".", "")
model_name = OpenAIConfig().get_azure_model_name(self.model_name)
response = client.completions.create(model=model_name, prompt=text_prompt,
temperature=self.temperature,
max_tokens=self.target_requested_output_tokens)
text_out = response.choices[0].text
usage = {"input": response.usage.prompt_tokens,
"output": response.usage.completion_tokens,
"total": response.usage.total_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
except Exception as e:
# catch error
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"OpenAIGenModel - inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes stream inference on OpenAI Model.
Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference.
"""
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if "openai_client" in inference_dict:
self.openai_client = inference_dict["openai_client"]
from llmware.configs import OpenAIConfig
if not self.openai_client:
azure_client = OpenAIConfig().get_azure_client()
else:
azure_client = self.openai_client
# api_key
if api_key:
self.api_key = api_key
if not self.api_key:
if not azure_client:
self.api_key = self._get_api_key()
if not self.api_key and not azure_client:
raise LLMWareException(message="OpenAIGenModel: no api_key found for OpenAI. This can be set as "
"an environment variable with: os.environ['OPENAI_API_KEY'] = '...'")
# call to preview hook (not implemented by default)
self.preview()
# default case - pass the prompt received without change
prompt_enriched = self.prompt
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
usage = {}
time_start = time.time()
try:
if self.model_name in ["o1-pro", "o3-mini"]:
# PATH #1 - the new 'responses' endpoint -> streaming not implemented yet
raise LLMWareException(message=f"Responses API streaming not implemented for this model. To use "
f"{self.model_name}, please use the .inference method")
elif self.model_name in ["gpt-5.2-pro", "gpt-5.2", "gpt-5-mini", "gpt-5-nano", "gpt-4.1"]:
messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict)
# updated OpenAI client to >v1.0 API - create client, and returns pydantic objects
if not azure_client:
client = OpenAI(api_key=self.api_key)
model_name = self.model_name
else:
logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig.")
client = azure_client
# adapt model name for azure, e.g., replace(".", "")
model_name = OpenAIConfig().get_azure_model_name(self.model_name)
text_out = ""
prompt_tokens = 0
completion_tokens = 0
total_tokens = 0
stream_response = client.chat.completions.create(model=model_name,messages=messages,
# max_tokens=self.target_requested_output_tokens,
stream=True)
# implement streaming generator to yield chunk of tokens
for chunk in stream_response:
if len(chunk.choices) > 0:
token = chunk.choices[0].delta.content or ""
text_out += token
yield token
usage = {"input": prompt_tokens,
"output": completion_tokens,
"total": prompt_tokens + completion_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
else:
# openai traditional 'instruct gpt' completion models
prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.separator
azure_client = OpenAIConfig().get_azure_client()
if not azure_client:
client = OpenAI(api_key=self.api_key)
model_name = self.model_name
else:
logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig.")
client = azure_client
model_name = OpenAIConfig().get_azure_model_name(self.model_name)
text_out = ""
prompt_tokens = 0
completion_tokens = 0
total_tokens = 0
stream_response = client.completions.create(model=model_name, prompt=text_prompt,
temperature=self.temperature,
max_tokens=self.target_requested_output_tokens,
stream=True)
# implement streaming generator to yield chunk of tokens
for chunk in stream_response:
if len(chunk.choices) > 0:
token = chunk.choices[0].delta.content or ""
text_out += token
yield token
usage = {"input": prompt_tokens,
"output": completion_tokens,
"total": prompt_tokens + completion_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
except Exception as e:
# catch error
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"OpenAIGenModel - OpenAI model inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class ClaudeModel(BaseModel):
""" ClaudeModel class implements the Anthropic Claude API for calling Anthropic models. """
def __init__(self, model_name=None, api_key=None, context_window=32768,
max_output=1000, temperature=0.0, **kwargs):
super().__init__(**kwargs)
self.model_class = "ClaudeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.api_key = api_key
if not api_key:
self.api_key = api_key
self.model_name = model_name
self.error_message = "\nUnable to connect to Anthropic/Claude. Please try again later."
self.separator = "\n"
# Claude/Anthropic model - 8000 max token context window
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings
if temperature >= 0.0:
self.temperature = temperature
else:
self.temperature = 0.0
self.target_requested_output_tokens = max_output
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
self.instruction_following = False
self.prompt_wrapper = None
if "model_card" in kwargs:
self.model_card = kwargs["model_card"]
else:
self.model_card = {}
self.post_init()
def set_api_key(self, api_key, env_var="ANTHROPIC_API_KEY"):
""" Utility method to set the API key in os.environ variable. """
os.environ[env_var] = api_key
logger.info(f"ClaudeModel - added and stored ANTHROPIC api_key in environmental variable- {env_var}")
return self
def _get_api_key(self, env_var="ANTHROPIC_API_KEY"):
""" Utility method to get api_key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error(f"ClaudeModel - _get_api_key could not successfully retrieve value from: {env_var}")
return self.api_key
def token_counter(self, text_sample):
""" Gets GPT2 tokenizer for fast approximate token counting. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer(self, query, context, inference_dict=None):
self.instruction_following = False
self.prompt_wrapper = False
# new
system_instruction = None
if inference_dict:
if "system_instruction" in inference_dict:
system_instruction = inference_dict["system_instruction"]
# end - new
# if loaded model was not pretrained on instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper,
instruction=system_instruction)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on Anthropic Model. Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
if not self.api_key:
raise LLMWareException(message=f"ClaudeModel - no api key found - you can set with: "
f"os.environ['ANTHROPIC_API_KEY'] = '...'")
# call to preview hook (not implemented by default)
self.preview()
try:
import anthropic
except ImportError:
raise DependencyNotInstalledException("anthropic")
client = anthropic.Client(api_key=self.api_key)
prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict)
time_start = time.time()
try:
# use messages API - older completion api is deprecated (and removed from ClaudeModel)
message = client.messages.create(model=self.model_name, max_tokens=self.target_requested_output_tokens,
messages=[{"role": "user", "content": prompt_enriched}] )
text_out = message.content[0].text
input_count = message.usage.input_tokens
output_count = message.usage.output_tokens
usage = {"input": input_count, "output": output_count, "total": input_count + output_count,
"metric": "tokens", "processing_time": time.time() - time_start}
except Exception as e:
# this is special error code that will be picked and handled by calling function
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"ClaudeModel - inference produced error - {e}")
output_response = {"llm_response": str(text_out), "usage": usage}
logger.debug(f"ClaudeModel - output_response - {output_response}")
# output inference parameters
self.llm_response = str(text_out)
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes streaming inference on Anthropic Model. Only required input is text-based prompt,
with optional parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens
configuration, and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
if not self.api_key:
raise LLMWareException(message=f"ClaudeModel - no api key found - you can set with: "
f"os.environ['ANTHROPIC_API_KEY'] = '...'")
# call to preview hook (not implemented by default)
self.preview()
try:
import anthropic
except ImportError:
raise DependencyNotInstalledException("anthropic")
client = anthropic.Client(api_key=self.api_key)
prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict)
time_start = time.time()
try:
# use messages API
message = client.messages.create(model=self.model_name, max_tokens=self.target_requested_output_tokens,
messages=[{"role": "user", "content": prompt_enriched}])
text_out = message.content[0].text
input_count = message.usage.input_tokens
output_count = message.usage.output_tokens
text_out = ""
prompt_tokens = 0
completion_tokens = 0
with client.messages.stream(
max_tokens=self.target_requested_output_tokens,
messages=[{"role": "user", "content": prompt_enriched}],
model=self.model_name) as stream:
for text in stream.text_stream:
# print(text, end="", flush=True)
text_out += text
yield text
usage = {"input": input_count, "output": output_count, "total": input_count + output_count,
"metric": "tokens", "processing_time": time.time() - time_start}
except Exception as e:
# this is special error code that will be picked and handled by calling function
text_out = "/***ERROR***/"
usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error(f"ClaudeModel inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
logger.debug(f"ClaudeModel - output_response - {output_response}")
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class GoogleGeminiModel(BaseModel):
""" GoogleGeminiModel class implements the current Google Gemini Model
API for calling Google Gemini models. """
def __init__(self, model_name=None, api_key=None, context_window=32768,
max_output=1000, temperature=0.0, **kwargs):
super().__init__(**kwargs)
self.model_class = "GoogleGeminiModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.api_key = api_key
if not api_key:
self.api_key = api_key
self.model_name = model_name
self.error_message = "\nUnable to connect to Google Gemini. Please try again later."
self.separator = "\n"
# Google Gemini model - 8000 max token context window
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings
if temperature >= 0.0:
self.temperature = temperature
else:
self.temperature = 0.0
self.target_requested_output_tokens = max_output
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
self.instruction_following = False
self.prompt_wrapper = None
self.post_init()
def set_api_key(self, api_key, env_var="GEMINI_API_KEY"):
""" Utility method to set the API key in os.environ variable. """
os.environ[env_var] = api_key
logger.info(f"GoogleGeminiModel - added and stored GOOGLE GEMINI api_key in "
f"environmental variable - {env_var}")
return self
def _get_api_key(self, env_var="GEMINI_API_KEY"):
""" Utility method to get api_key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error(f"GoogleGeminiModel - _get_api_key could not successfully "
f"retrieve value from: {env_var}")
return self.api_key
def prompt_engineer(self, query, context, inference_dict=None):
self.instruction_following = False
self.prompt_wrapper = False
system_instruction = None
if inference_dict:
if "system_instruction" in inference_dict:
system_instruction = inference_dict["system_instruction"]
# if loaded model was not pretrained on instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper,
instruction=system_instruction)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on Google Gemini Model. Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
if not self.api_key:
logger.warning("GoogleGeminiModel - inference - invoking "
"Google Gemini Generative model with no api_key")
return False
# call to preview hook (not implemented by default)
self.preview()
try:
from google import genai
from google.genai import types
except ImportError:
raise DependencyNotInstalledException("google")
client = genai.Client(
api_key=self.api_key,
http_options=types.HttpOptions(api_version='v1alpha')
)
prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict)
time_start = time.time()
try:
response = client.models.generate_content(
model=self.model_name, contents=prompt_enriched)
text_out = response.text
input_count = response.usage_metadata.prompt_token_count
output_count = response.usage_metadata.total_token_count
usage = {"input": input_count, "output": output_count,
"total": input_count + output_count,
"metric": "tokens",
"processing_time": time.time() - time_start}
except Exception as e:
# this is special error code that will be picked and handled by calling function
text_out = "/***ERROR***/"
usage = {"input":0, "output":0, "total":0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.warning(f"GoogleGeminiModel - inference produced error - {e}")
output_response = {"llm_response": str(text_out), "usage": usage}
# output inference parameters
self.llm_response = str(text_out)
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
def _prep_gemini_img_file(self, image_fp):
""" Utility function to prepare image for processing by Gemini """
try:
from google import genai
from google.genai import types
except ImportError:
raise DependencyNotInstalledException("google")
img = open(image_fp, "rb").read()
ext = image_fp.split(".")[-1]
if ext in ["jpg", "jpeg"]:
mime_type = "image/jpeg"
elif ext in ["png"]:
mime_type = "image/png"
else:
mime_type = "image/jpeg"
img_content = types.Part.from_bytes(data=img, mime_type=mime_type)
return img_content
def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None, image_files=None, doc_files=None):
""" Executes streaming inference on Gemini Model. Only required input is text-based prompt,
with optional parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens
configuration, and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
if not self.api_key:
raise LLMWareException("GoogleGeminiModel - no api_key found - you can set with: "
"os.environ['GEMINI_API_KEY'] = '...'")
# call to preview hook (not implemented by default)
self.preview()
try:
from google import genai
from google.genai import types
except ImportError:
raise DependencyNotInstalledException("google")
client = genai.Client(
api_key=self.api_key,
http_options=types.HttpOptions(api_version='v1alpha')
)
prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict)
time_start = time.time()
content = []
content.append(prompt_enriched)
if image_files:
for img_fp in image_files:
img_content = self._prep_gemini_img_file(img_fp)
content.append(img_content)
try:
for chunk in client.models.generate_content_stream(model=self.model_name,
contents=content):
yield chunk.text
text_out = ""
prompt_tokens = 0
completion_tokens = 0
usage = {"input": prompt_tokens, "output": completion_tokens,
"total": prompt_tokens + completion_tokens,
"metric": "tokens", "processing_time": time.time() - time_start}
except Exception as e:
# this is special error code that will be picked and handled by calling function
text_out = "/***ERROR***/"
usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.warning(f"GoogleGeminiModel - streaming inference produced error - {e}")
output_response = {"llm_response": text_out, "usage": usage}
logger.debug(f"GoogleGeminiModel - output_response - {output_response}")
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class ONNXQNNGenerativeModel(BaseModel):
"""ONNXQNNGenerativeModel class implements the ONNX generative model API in conjunction
with QNN execution provider to access NPU on Windows Arm 64.
note: this code and associated prepackaged models are pinned to the
following specific versions:
-- pip install onnxruntime-qnn==1.22.2
-- pip install onnxruntime-genai==0.9.0
... built with qnn sdk 2.36.1
... running on Windows Arm 64 Qualcomm Snapdragon NPU
... does not currently support Android - but is on the roadmap
"""
def __init__(self, model_name=None, api_key=None, model_card=None,
prompt_wrapper=None, instruction_following=False, context_window=2048,
use_gpu_if_available=True, trust_remote_code=True, sample=True, max_output=100, temperature=0.3,
get_logits=False, api_endpoint=None, **kwargs):
super().__init__()
self.model_class = "ONNXQNNGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
logger.info(f"ONNXQNNGenerativeModel - starting constructor with model - {model_name}")
# pull in expected hf input
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = None
self.tokenizer = None
self.generator = None
self.sample = sample
self.get_logits = get_logits
self.auto_remediate_function_call_output = True
# Function Call parameters
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
self.tool_type = None
self.npu_optimized = False
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
if "npu_optimized" in model_card:
self.npu_optimized = True
# instantiate if model_name passed without actual model and tokenizer
if model_name and not api_endpoint:
hf_repo_name = self.model_name
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(self.model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
self.hf_tokenizer_name = hf_repo_name
self.model = None
self.tokenizer = None
self.tokenizer_stream = None
# set to defaults for HF models in Model Catalog
# this can be over-ridden post initiation if needed for custom models
self.prompt_wrapper = "human_bot"
self.instruction_following = False
self.params = None
# set specific parameters associated with custom models
# note - these two parameters will control how prompts are handled - model-specific
self.prompt_wrapper = "human_bot"
self.instruction_following = instruction_following
if not model_card:
# safety - empty iterable rather than 'None'
model_card = []
# deprecated attribute - will be removed in future releases
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
# loads onnxruntime_genai, which in turn will look for backend qnn implementation
# please ensure that onnxruntime_qnn has been imported into the project
# onnxruntime_qnn==1.22.2
global GLOBAL_ONNX_GENAI_RUNTIME
if not GLOBAL_ONNX_GENAI_RUNTIME:
if util.find_spec("onnxruntime_genai"):
try:
global og
og = importlib.import_module("onnxruntime_genai")
GLOBAL_ONNX_GENAI_RUNTIME = True
except:
raise LLMWareException(message="ONNXQNNGenerativeModel: could not load onnxruntime_genai module. "
"To fix: please check the following:\n"
"1. pip install onnxruntime_qnn==1.22.2\n"
"2. pip install onnxruntime_genai==0.9.0\n"
"3. confirm Windows Arm64 with Snapdragon NPU")
# sets trailing space default when constructing the prompt
# in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves
# performance
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = context_window
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
# key output parameters
self.max_output = max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# use 0 as eos token id by default in generation -> but try to pull from model config
self.eos_token_id = 0
self.use_gpu = False
# coming soon
self.windows_local_foundry_active = False
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load HuggingFace model."
# temperature settings
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.3
self.temperature = 0.3
self.add_prompt_engineering = False
self.add_context = ""
self.context = ""
self.prompt = ""
# not currently implemented for this model class
self.api_endpoint = api_endpoint
self.model_repo_path = None
# confirm platform check
import sys
import platform
plat = sys.platform
mach = platform.machine().lower()
logger.info(f"ONNXQNNGenerativeModel - platform - {plat} - machine - {mach}")
if not (plat == "win32" and mach == "arm64"):
logger.warning(f"ONNXQNNGenerativeModel is designed for Windows Arm64.")
self.post_init()
def load_model_for_inference(self, loading_directions, model_card=None):
""" Loads ONNX Model from local path using loading directions. """
self.model_repo_path = loading_directions
if model_card:
self.model_card = model_card
self.validate()
onnx_model_path = os.path.join(LLMWareConfig().get_model_repo_path(),
self.model_name)
if self.npu_optimized:
# get npu optimized onnxruntime with qnn
set_for_npu_qnn = True
# starting with onnxruntime-qnn 2.0, need to set qnn execution provider path
# e.g., path to "onnxruntime_providers_qnn.dll"
qnn_path = os.environ.get("qnn_onnx_path","")
if not qnn_path:
# by default, look in the onnxruntime_qnn package
import onnxruntime_qnn
backend_path = os.path.dirname(onnxruntime_qnn.__file__)
qnn_path = os.path.join(backend_path, "onnxruntime_providers_qnn.dll")
# register the backend
og.register_execution_provider_library("QNNExecutionProvider", qnn_path)
logger.info(f"ONNXQNNGenerativeModel - load_model_for_inference - qnn path - {qnn_path}")
# use global onnxruntime_genai - constructing model from config
config = og.Config(onnx_model_path)
self.model = og.Model(config)
self.tokenizer = og.Tokenizer(self.model)
self.tokenizer_stream = self.tokenizer.create_stream()
search_options = {}
search_options['max_length'] = 2048
search_options['batch_size'] = 1
self.params = og.GeneratorParams(self.model)
self.params.set_search_options(**search_options)
logger.info(f"ONNXQNNGenerativeModel - constructed model - {self.model_name}.")
return self
def unload_model(self):
""" Not implemented. """
return True
def set_api_key(self, api_key, env_var=""):
""" Not implemented for ONNXQNNGenerativeModel """
return True
def _get_api_key(self, env_var=""):
""" Not implemented for ONNXQNNGenerativeModel """
return True
def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None):
""" Executes generation inference on model. """
# first prepare the prompt
t0 = time.time()
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.trailing_space
input_tokens = self.tokenizer.encode(text_prompt)
token_count = 0
output = ""
generator = og.Generator(self.model, self.params)
# note: onnxruntime_genai library makes a lot of small breaking changes
# in their generation loops -> this should be OK with versions >0.9.0
# if you see error, then check the documentation for onnxruntime_genai
# which is pretty good at explaining/documenting the change and how to fix
generator.append_tokens(input_tokens)
try:
while not generator.is_done():
token_count += 1
# change in v0.6 api - explicit compute logits call not required
# generator.compute_logits()
generator.generate_next_token()
# not activated currently
self.get_logits = False
# to get logit value
if self.get_logits:
logit = generator.get_output("logits")
self.register_top_logits(logit)
new_token = generator.get_next_tokens()[0]
if self.get_logits:
self.output_tokens.append(new_token)
output += self.tokenizer_stream.decode(new_token)
if token_count > self.max_output:
break
except Exception as e:
logger.warning(f"ONNXQNNGenerativeModel inference produced error - {e}")
pass
del generator
usage = {"input": len(input_tokens),
"output": token_count,
"total": len(input_tokens) + token_count,
"metric": "tokens",
"processing_time": time.time() - t0}
output_response = {"llm_response": output, "usage": usage}
if self.get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None, skip_pe_override=False):
""" Executes stream generation inference on model. """
# first prepare the prompt
t0 = time.time()
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
self.preview()
text_prompt = self.prompt
if self.add_prompt_engineering and not skip_pe_override:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
text_prompt = prompt_final + self.trailing_space
logger.debug("ONNXQNNGenerative Model - onnx stream starting.")
input_tokens = self.tokenizer.encode(text_prompt)
token_count = 0
output = ""
# note: onnxruntime_genai library makes a lot of small breaking changes
# in their generation loops -> this should be OK with versions > 0.9.0
# if you see error, then check the documentation for onnxruntime_genai
# which is pretty good at explaining/documenting the change and how to fix
self.generator = og.Generator(self.model, self.params)
self.generator.append_tokens(input_tokens)
while True:
token_count += 1
# change in v0.6 api - no explicit compute logits call
# self.generator.compute_logits()
self.generator.generate_next_token()
if self.generator.is_done():
break
self.get_logits = False
# to get logit value
if self.get_logits:
logit = self.generator.get_output("logits")
self.register_top_logits(logit)
new_token = self.generator.get_next_tokens()[0]
if self.get_logits:
self.output_tokens.append(new_token)
output += self.tokenizer_stream.decode(new_token)
if token_count > self.max_output:
break
yield self.tokenizer_stream.decode(new_token)
self.generator = None
usage = {"input": len(input_tokens),
"output": token_count,
"total": len(input_tokens) + token_count,
"metric": "tokens",
"processing_time": time.time() - t0}
output_response = {"llm_response": output, "usage": usage}
if self.get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output
self.usage = usage
self.final_prompt = text_prompt
self.register()
logger.debug("ONNXQNNGenerativeModel - completed stream generation.")
return output_response
def cleanup_stream_gen_on_early_stop(self):
self.generator = None
return True
def register_top_logits(self, logit):
""" Gets the top logits and keeps a running log for output analysis. """
# logit will be in form of (1,1,vocab_len), for all but the first logit
# if first logit (will have shape of context len - add [-1])
if logit.shape[1] > 1:
# used for first logit with shape, e.g., (1,input_token_len,vocab_size)
logit_array = logit.squeeze()[-1]
else:
# all other logits after the first token
logit_array = logit.squeeze()
logit_size = logit.shape[-1]
# useful check on shape of logit_array
logit_array_size = logit_array.shape
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
for x in range(0, self.top_logit_count):
# round the float number to 3 digits
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3))
top_logits.append(pair)
self.logits_record.append(top_logits)
return top_logits
class LLMWareModel(BaseModel):
"""LLMWareModel class implements the API for LLMWare generative models. """
def __init__(self, model_name=None, api_key=None, context_window=2048, **kwargs):
super().__init__(**kwargs)
self.model_class = "LLMWareModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.api_key = api_key
self.model_name = model_name
self.model = None
self.tokenizer = None
self.error_message = "\nUnable to connect to LLMWare GPT API. Please try again later."
# set max_total_len -> adjust input and output based on use case
self.max_total_len = context_window
self.max_input_len = int(0.4 * context_window)
self.llm_max_output_len = int(0.4 * context_window)
self.separator = "\n"
# inference settings
self.temperature = 0.2
self.target_requested_output_tokens = 200
self.add_prompt_engineering = True
self.add_context = ""
self.prompt = ""
self.post_init()
def set_api_key(self, api_key, env_var="USER_MANAGED_LLMWARE_GPT_API_KEY"):
""" Utility method to set the API key in os.environ variable. """
os.environ[env_var] = api_key
logger.info("update: added and stored READ_GPT api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_LLMWARE_GPT_API_KEY"):
""" Utility method to get api_key from os.environ variable. """
self.api_key = os.environ.get(env_var)
return self.api_key
def token_counter(self, text_sample):
""" Gets GPT2 tokenizer for fast approximate token counting. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer(self, query, context, inference_dict=None):
""" Builds prompt by assembling query, context and applying the selected prompt style. """
if not query:
query = "What is a list that summarizes the key points?"
# default_case
prompt_engineered = context + "\n" + query
if self.add_prompt_engineering == "top_level_summary_select":
prompt_engineered += query + "\n"
prompt_engineered += "Which of the following selections best answers the question?"
prompt_engineered += context
if self.add_prompt_engineering == "summarize_with_bullets_no_query":
issue = "What is a list of the most important points?"
prompt_engineered = context + "\n" + issue
return prompt_engineered
def load_model_for_inference(self, model_name=None, model_card=None,fp=None, **kwargs):
# validate before loading - turned off
# self.validate()
# look up model_name in configs
if model_name:
self.model_name = model_name
return self
def load_pretrained_model(self, model_name=None):
if model_name:
self.model_name = model_name
# convenience method for pretrained models as a single step
return self
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on LLMWare Model. Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# call to preview hook (not implemented by default)
self.preview()
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
# safety check on length - set cap with small 'buffer'
input_tokens = self.token_counter(prompt_enriched)
buffer = 10
available_tokens_in_output_context_window = self.max_total_len - input_tokens - buffer
# if target requested output is less, then keep - otherwise, cap with 'safe' maximum len
target_len = min(self.target_requested_output_tokens, available_tokens_in_output_context_window)
output_dict_new = {}
output_response = {}
usage = {"input": input_tokens}
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
params = {"context": self.add_context,
"question": self.prompt,
"max_output_tokens": target_len,
"api_key": self.api_key}
# params = {"context": prompt["context"],"question": prompt["query"], "max_output_tokens": 50, "api_key": good_key}
time_start = time.time()
try:
output = requests.post(os.environ.get("LLMWARE_GPT_URI"), data=params)
output_dict_new = ast.literal_eval(output.text)
success_path = 1
output_response = output_dict_new
except:
text_output = "/***ERROR***/"
usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens",
"processing_time": time.time() - time_start}
logger.error("error: no response from aib remote server for llmware-gpt model - "
"check api key and connection")
success_path = -1
output_response = {"llm_response": "", "usage": usage}
# output inference parameters
self.llm_response = ""
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class OpenAIEmbeddingModel(BaseModel):
""" OpenAIEmbeddingModel class implements the OpenAI API for embedding models. """
def __init__(self, model_name=None, api_key=None, embedding_dims=None, model_card=None, max_len=None, **kwargs):
super().__init__(**kwargs)
self.model_class = "OpenAIEmbeddingModel"
self.model_category = "embedding"
# must have elements for embedding model
self.model_name = model_name
self.api_key = api_key
self.model_card = model_card
self.tokenizer = None
if not embedding_dims:
self.embedding_dims = 1536
else:
self.embedding_dims = embedding_dims
# openai standard for embeddings is 8191 as of feb 2024
self.max_total_len = 8191
self.max_len = self.max_total_len
if model_card:
if "embedding_dims" in model_card:
self.embedding_dims = model_card["embedding_dims"]
if "context_window" in model_card:
self.max_total_len = model_card["context_window"]
self.error_message = "\nUnable to connect to OpenAI. Please try again later."
if max_len:
if max_len < self.max_total_len:
self.max_len = max_len
self.text_sample = None
self.post_init()
def set_api_key(self, api_key,env_var="USER_MANAGED_OPENAI_API_KEY"):
""" Utility method to set the API key in os.environ variable. """
os.environ[env_var] = api_key
logger.info("update: added and stored OpenAI api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_OPENAI_API_KEY"):
""" Utility method to get api_key from os.environ variable. """
self.api_key = os.environ.get(env_var)
return self.api_key
def get_tokenizer(self):
self.tokenizer = Utilities().get_default_tokenizer()
return self.tokenizer
def token_counter(self, text_sample):
""" Counts tokens in text sample. """
return len(self.tokenizer.encode(text_sample).ids)
def embedding(self, text_sample, api_key=None):
self.text_sample = text_sample
# call to preview (not implemented by default)
self.preview()
if api_key:
self.api_key = api_key
if not self.api_key:
self.api_key = self._get_api_key()
if not self.api_key:
logger.error("error: invoking OpenAI Embedding model with no api_key")
# need to prepare for batches
if isinstance(self.text_sample, list):
text_prompt = self.text_sample
input_len = len(text_sample)
else:
text_prompt = [self.text_sample]
input_len = 1
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
from llmware.configs import OpenAIConfig
# insert safety check here
safe_samples = []
safety_buffer = 200
if self.max_total_len < 8191:
self.max_total_len = 8191
tokenizer = self.get_tokenizer()
for sample in text_prompt:
tok_len = self.token_counter(sample)
if tok_len < (self.max_total_len - safety_buffer):
safe_samples.append(sample)
else:
if len(sample) > 300:
display_sample = sample[0:300] + " ... "
else:
display_sample = sample
logger.warning(f"warning: OpenAI Embedding - input sample len - {tok_len} > context_window size "
f"\ninput_sample - {display_sample} "
f"\n\nSample is being truncated.")
tok = tokenizer.encode(sample).ids
tok = tok[0:(self.max_total_len - safety_buffer)]
sample = tokenizer.decode(tok)
safe_samples.append(sample)
text_prompt = safe_samples
# end - safety check
# update to open >v1.0 api - create client and output is pydantic objects
azure_client = OpenAIConfig().get_azure_client()
if not azure_client:
client = OpenAI(api_key=self.api_key)
else:
logger.info("update: applying custom OpenAI client from OpenAIConfig")
client = azure_client
response = client.embeddings.create(model=self.model_name, input=text_prompt)
if input_len == 1:
embedding = response.data[0].embedding
else:
embedding = []
for i, entries in enumerate(response.data):
embedding.append(response.data[i].embedding)
self.register()
return embedding
class HFReRankerModel(BaseModel):
"""HFReRankerModel class implements the interface for HuggingFace ReRanker models. """
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
embedding_dims=None, trust_remote_code=False, use_gpu_if_available=True, max_len=None, **kwargs):
super().__init__(**kwargs)
self.model_class = "HFReRankerModel"
self.model_category = "reranker"
# pull in expected hf input
self.model_name = model_name
self.model = model
self.tokenizer= tokenizer
self.embedding_dims = embedding_dims
self.model_type = None
self.max_total_len = 2048
self.model_architecture = None
self.model_card = model_card
self.safe_buffer = 12
# default for HF embedding model -> will be over-ridden by model card / configs, if available
self.context_window = 512
if self.model_card:
if "embedding_dims" in self.model_card:
self.embedding_dims = self.model_card["embedding_dims"]
if "context_window" in self.model_card:
self.context_window = self.model_card["context_window"]
# insert dynamic pytorch load here
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
logger.debug("update: ModelCatalog - HFReRankerModel - local dynamic load of torch here")
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
# end dynamic import here
if self.model_name and not model:
# pull from HF
hf_repo_name = self.model_name
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
pt_loader = PyTorchLoader(api_key=api_key,trust_remote_code=trust_remote_code,custom_loader=None)
self.model=pt_loader.get_reranker_model(hf_repo_name)
self.tokenizer=None
self.use_gpu = torch.cuda.is_available() and use_gpu_if_available
if self.model:
self.config = self.model.config.to_dict()
if "hidden_size" in self.config:
self.embedding_dims = self.config["hidden_size"]
if "model_type" in self.config:
self.model_type = self.config["model_type"]
if "max_position_embeddings" in self.config:
try:
self.context_window = int(self.config["max_position_embeddings"])
except:
pass
if "_name_or_path" in self.config:
self.model_name = self.config["_name_or_path"]
if "architectures" in self.config:
if isinstance(self.config["architectures"],list):
self.model_architectures = self.config["architectures"][0]
else:
self.model_architectures = self.config["architectures"]
self.model.eval()
if self.use_gpu:
self.model.to('cuda')
else:
raise ModelNotFoundException(model_name)
# no api key expected or required
self.api_key = api_key
# set max len for tokenizer truncation with 'safe_buffer' below context_window size
if self.context_window > self.safe_buffer:
self.max_len = self.context_window - self.safe_buffer
else:
self.max_len = self.context_window
# option to set smaller size than model context window
if max_len:
if max_len < self.context_window:
self.max_len = max_len
self.query = ""
self.text_results = None
self.post_init()
def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"):
""" Sets the API key - generally not needed for public HF repositories. """
os.environ[env_var] = api_key
logger.info("update: added and stored HF api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var)
return self.api_key
def token_counter(self, text_sample):
""" Counts tokens in text sample. Not currently implemented. """
return -1
def inference (self, query, text_results, api_key=None, top_n=20, relevance_threshold=None, min_return=3):
""" Executes reranking inference. """
self.query = query
self.text_results = text_results
# call to preview (not implemented by default)
self.preview()
documents = []
for i, chunks in enumerate(text_results):
documents.append(chunks['text'])
sentence_pairs = [[self.query, doc] for doc in documents]
scores = self.model.compute_score(sentence_pairs)
output = []
for i, score in enumerate(scores):
text_results[i].update({"rerank_score": score})
output.append(text_results[i])
ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True)
# will return top_n if no relevance threshold set
if not relevance_threshold:
if top_n < len(ranked_output):
final_output = ranked_output[0:top_n]
else:
final_output = ranked_output
else:
final_output = []
# if relevance threshold, will return all results above threshold
for entries in ranked_output:
if entries["rerank_score"] >= relevance_threshold:
final_output.append(entries)
# fallback, if no result above threshold, then will return the min number of results
if len(final_output) == 0:
final_output = ranked_output[0:min_return]
self.register()
return final_output
class HFEmbeddingModel(BaseModel):
"""HFEmbeddingModel class implements the API for HuggingFace embedding models. """
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
embedding_dims=None, trust_remote_code=False, use_gpu_if_available=True, max_len=None, **kwargs):
super().__init__(**kwargs)
self.model_class = "HFEmbeddingModel"
self.model_category = "embedding"
# pull in expected hf input
self.model_name = model_name
self.model = model
self.tokenizer= tokenizer
self.embedding_dims = embedding_dims
self.model_type = None
self.max_total_len = 2048
self.model_architecture = None
self.model_card = model_card
self.safe_buffer = 12
# default for HF embedding model -> will be over-ridden by model card / configs, if available
self.context_window = 512
if self.model_card:
if "embedding_dims" in self.model_card:
self.embedding_dims = self.model_card["embedding_dims"]
if "context_window" in self.model_card:
self.context_window = self.model_card["context_window"]
# insert dynamic pytorch load here
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
logger.debug("update: ModelCatalog - HFEmbeddingModel - local dynamic load of torch here")
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
# end dynamic import here
if self.model_name and not model:
# pull from HF
hf_repo_name = self.model_name
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
pt_loader = PyTorchLoader(api_key=api_key,trust_remote_code=trust_remote_code,custom_loader=None)
self.model=pt_loader.get_embedding_model(hf_repo_name)
self.tokenizer=pt_loader.get_tokenizer(hf_repo_name)
self.use_gpu = torch.cuda.is_available() and use_gpu_if_available
if self.model:
self.config = self.model.config.to_dict()
if "hidden_size" in self.config:
self.embedding_dims = self.config["hidden_size"]
if "model_type" in self.config:
self.model_type = self.config["model_type"]
if "max_position_embeddings" in self.config:
try:
self.context_window = int(self.config["max_position_embeddings"])
except:
pass
if "_name_or_path" in self.config:
self.model_name = self.config["_name_or_path"]
if "architectures" in self.config:
if isinstance(self.config["architectures"],list):
self.model_architectures = self.config["architectures"][0]
else:
self.model_architectures = self.config["architectures"]
self.model.eval()
if self.use_gpu:
self.model.to('cuda')
else:
raise ModelNotFoundException(model_name)
# no api key expected or required
self.api_key = api_key
# set max len for tokenizer truncation with 'safe_buffer' below context_window size
if self.context_window > self.safe_buffer:
self.max_len = self.context_window - self.safe_buffer
else:
self.max_len = self.context_window
# option to set smaller size than model context window
if max_len:
if max_len < self.context_window:
self.max_len = max_len
self.text_sample = None
self.post_init()
def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"):
""" Sets the API key - generally not needed for public HF repositories. """
os.environ[env_var] = api_key
logger.info("update: added and stored HF api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var)
return self.api_key
def token_counter(self, text_sample):
""" Counts tokens in text sample. """
# need to support HF tokenizer
toks = self.tokenizer.encode(text_sample).ids
return len(toks)
def embedding (self, text_sample, api_key=None):
""" Executes embedding inference. """
self.text_sample = text_sample
# call to preview (not implemented by default)
self.preview()
# return embeddings only
if isinstance(self.text_sample,list):
sequence = self.text_sample
else:
sequence = [self.text_sample]
model_inputs = self.tokenizer(sequence, truncation=True, max_length=self.max_len, return_tensors="pt",padding=True)
if self.use_gpu:
input_ids = model_inputs.input_ids.to('cuda')
attn_mask = model_inputs.attention_mask.to('cuda')
else:
input_ids = model_inputs.input_ids.to('cpu')
attn_mask = model_inputs.attention_mask.to('cpu')
# context manager to run inference without saving/calculating grads
with torch.no_grad():
model_outputs = self.model(input_ids, attention_mask=attn_mask)
embedding = model_outputs.last_hidden_state[:,0]
# normalize hf embeddings
embeddings_normalized = torch.nn.functional.normalize(embedding, p=2, dim=1)
if self.use_gpu:
embeddings_normalized = np.array(embeddings_normalized.detach().to('cpu'))
else:
embeddings_normalized = embeddings_normalized.detach().numpy()
self.register()
return embeddings_normalized
class HFGenerativeModel(BaseModel):
""" HFGenerativeModel class implements the HuggingFace generative model API, and is used generally for
models in HuggingFace repositories, e.g., Dragon, Bling, etc. """
# support instantiating HF model in two different ways:
# 1. directly passing a previously loaded HF model object and tokenizer object
# 2. passing a model_name only, which will then create the model and tokenizer
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
prompt_wrapper=None, instruction_following=False, context_window=2048,
use_gpu_if_available=True, trust_remote_code=True, sample=True,max_output=100, temperature=0.3,
get_logits=False, api_endpoint=None, **kwargs):
super().__init__(**kwargs)
self.model_class = "HFGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
# pull in expected hf input
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = model
self.tokenizer = tokenizer
# new parameters
self.sample=sample
self.get_logits=get_logits
self.auto_remediate_function_call_output = True
# Function Call parameters
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
# insert dynamic pytorch load here
if not api_endpoint:
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
# end dynamic import here
# instantiate if model_name passed without actual model and tokenizer
if model_name and not model and not tokenizer and not api_endpoint:
hf_repo_name = self.model_name
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(self.model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
self.hf_tokenizer_name = hf_repo_name
pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=trust_remote_code, custom_loader=None)
self.model = pt_loader.get_generative_model(hf_repo_name)
self.tokenizer = pt_loader.get_tokenizer(hf_repo_name)
# set to defaults for HF models in Model Catalog
# this can be over-ridden post initiation if needed for custom models
self.prompt_wrapper = "human_bot"
self.instruction_following = False
# set specific parameters associated with custom models
# note - these two parameters will control how prompts are handled - model-specific
self.prompt_wrapper = prompt_wrapper
self.instruction_following = instruction_following
if not model_card:
# safety - empty iterable rather than 'None'
model_card = []
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
# sets trailing space default when constructing the prompt
# in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves
# performance
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = context_window
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
# key output parameters
self.max_output=max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# use 0 as eos token id by default in generation -> but try to pull from model config
self.eos_token_id = 0
# will load model and inference onto gpu,
# if (a) CUDA available and (b) use_gpu_if_available set to True (default)
if not api_endpoint:
self.use_gpu = torch.cuda.is_available() and use_gpu_if_available
else:
self.use_gpu = False
if self.model:
if isinstance(self.model.config, dict):
self.config = self.model.config
else:
self.config = self.model.config.to_dict()
if "trailing_space" in self.config:
self.trailing_space = self.config["trailing_space"]
if "eos_token_id" in self.config:
# only use to set if value is not None
if self.config["eos_token_id"]:
self.eos_token_id = self.config["eos_token_id"]
if "model_type" in self.config:
self.model_type = self.config["model_type"]
if "hidden_size" in self.config:
self.embedding_dims = self.config["hidden_size"]
if "max_position_embeddings" in self.config:
self.max_total_len = self.config["max_position_embeddings"]
if "architectures" in self.config:
if isinstance(self.config["architectures"], list):
self.model_architectures = self.config["architectures"][0]
else:
self.model_architectures = self.config["architectures"]
# prepare model for inference
self.model.eval()
if self.use_gpu:
self.model.to('cuda')
logger.debug("update: HFGenerative loading - moving model to cuda")
else:
if not api_endpoint:
logger.error("error: HFGenerativeModel - could not identify model - ", model_name)
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load HuggingFace model."
# temperature settings
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.3
self.temperature = 0.3
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
self.context = ""
self.tool_type = None
self.api_endpoint = api_endpoint
self.post_init()
def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"):
""" Sets the API key - generally not needed for public HF repositories. """
os.environ[env_var] = api_key
logger.info("update: added and stored HF api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"):
""" Gets API key from os.environ variable. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var)
return self.api_key
def token_counter(self, text_sample):
""" Quick approximate token counter - uses default tokenizer so may have minor differences from the
model's actual tokenization. """
tokenizer = Utilities().get_default_tokenizer()
toks = tokenizer.encode(text_sample).ids
return len(toks)
def prompt_engineer(self, query, context, inference_dict):
""" Applies prompt and templating preparation. """
# if loaded model was not pretrained on instruction_following, then skip any instructions
if not self.instruction_following:
if context:
output = context + "\n" + query
else:
output = query
# unlikely that there would be an 'instruct wrapping' on text, but allow for possibility
if self.prompt_wrapper:
output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper,
instruction=None)
return output
# move ahead to add instructions and prompt engineering
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
selected_prompt = self.add_prompt_engineering
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query,
context=context,
inference_dict=inference_dict)
if prompt_dict:
prompt_engineered = prompt_dict["core_prompt"]
else:
# default case
prompt_engineered = "Please read the following text: " + context + self.separator
prompt_engineered += "Based on this text, please answer the question: " + query + self.separator
prompt_engineered += "Please answer the question only with facts provided in the materials. " \
"If the question can not be answered in the materials, then please " \
"respond 'Not Found.'"
# final wrapping, based on model-specific instruct training format
# --provides a final 'wrapper' around the core prompt text, based on model expectations
if self.prompt_wrapper:
prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper,
instruction=None)
return prompt_engineered
def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None):
""" Executes generation inference on model. """
self.prompt = prompt
# first prepare the prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# add defaults if add_prompt_engineering not set
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - defaults update
# show warning if function calling model
if self.fc_supported:
logger.warning("This is a function calling model - using .inference may lead to unexpected "
"results. Recommended to use the .function_call method to ensure correct prompt "
"template packaging.")
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# call to preview (not implemented by default)
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# text_prompt = prompt_final + "\n"
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
text_prompt = prompt_final + self.trailing_space
# second - tokenize to get the input_ids
tokenizer_output = self.tokenizer.encode(text_prompt)
input_token_len = len(tokenizer_output)
input_ids = torch.tensor(tokenizer_output).unsqueeze(0)
# explicit check and setting to facilitate debugging
if self.use_gpu:
input_ids = input_ids.to('cuda')
else:
input_ids = input_ids.to('cpu')
# time start
time_start = time.time()
# This simplified greedy sampling generation loop was derived from and inspired by ideas in the
# HuggingFace transformers library generation class.
# https: //github.com/huggingface/transformers/tree/main/src/transformers/generation
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc.team, and NVIDIA Corporation.
# Licensed under the Apache License, Version 2.0 (the "License")
# default settings
pad_token_id = 0
# for most models, eos_token_id = 0, but llama and mistral = 2
eos_token_id = [self.eos_token_id]
# eos_token_id = [0]
eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device)
# keep track of which sequences are already finished
unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
this_peer_finished = False # used by synced_gpus only
# auto-regressive generation
new_tokens_generated = 0
attn_mask = torch.ones(input_ids.shape[1]).unsqueeze(0)
# explicit check and setting to facilitate debugging, if needed
if self.use_gpu:
attn_mask = attn_mask.to('cuda')
else:
attn_mask = attn_mask.to('cpu')
batch_size = input_ids.shape[0]
seq_len = input_ids.shape[1]
pkv = None
# borrow setting from GGUFConfigs
get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed")
t_gen_start = time.time()
first_token_processing_time = -1.0
while True:
inp_one_time: torch.LongTensor = input_ids
if new_tokens_generated > 0:
inp_one_time = input_ids[:, -1:]
# explicit check and setting to facilitate debugging, if needed
if self.use_gpu:
inp0 = inp_one_time.to('cuda')
inp1 = attn_mask.to('cuda')
else:
inp0 = inp_one_time.to('cpu')
inp1 = attn_mask.to('cpu')
# inp3 = torch.LongTensor([new_tokens_generated])
# need to invoke forward pass on model
# outputs = self.model(inp0,inp1,pkv)
# context manager to avoid saving/computing grads in forward pass
with torch.no_grad():
outputs = self.model(input_ids=inp0, attention_mask=inp1, past_key_values=pkv,
return_dict=True)
if new_tokens_generated == 0:
if get_first_token_speed:
first_token_processing_time = time.time() - t_gen_start
new_tokens_generated += 1
next_token_logits = outputs.logits[:, -1, :]
# capture top logits - not currently activated for inference
# self.register_top_logits(next_token_logits)
# shape of next_token_logits = torch.Size([1, 32000])
# logger.debug(f"next token logits shape - {next_token_logits.shape}")
if self.temperature and self.sample:
next_token_scores = next_token_logits / self.temperature
else:
next_token_scores = next_token_logits
# get token from logits
probs = torch.nn.functional.softmax(next_token_scores, dim=-1)
if not self.sample:
# will pull the 'top logit' only
next_tokens = torch.argmax(probs).unsqueeze(0)
else:
# will apply probabilistic sampling
next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
# new - option to capture logits and output tokens for analysis
if self.get_logits:
self.register_top_logits(next_token_logits)
# capture the output tokens
if self.use_gpu:
next_tokens_np = np.array(next_tokens.to('cpu'))
else:
next_tokens_np = np.array(next_tokens)
self.output_tokens.append(next_tokens_np[0])
# finished sentences should have their next token be a padding token
if eos_token_id is not None:
next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences)
# update generated ids, model inputs, and length for next step
input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
# testing output in progress starts here
"""
logger.debug(f"update: input_ids - {input_ids}")
# outputs_detached = outputs.to('cpu')
outputs_np = np.array(input_ids[0])
output_str = self.tokenizer.decode(outputs_np)
logger.debug(f"update: output string - {output_str}")
"""
# end - testing output in progress
pkv = outputs.past_key_values
# update attention mask
attn_mask = torch.cat([attn_mask, attn_mask.new_ones((attn_mask.shape[0], 1))], dim=-1)
# if eos_token was found in one sentence, set sentence to finished
if eos_token_id_tensor is not None:
unfinished_sequences = unfinished_sequences.mul(
next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
)
# stop when each sentence is finished
if unfinished_sequences.max() == 0:
this_peer_finished = True
# stop if we exceed the maximum length
if new_tokens_generated >= self.target_requested_output_tokens:
this_peer_finished = True
if this_peer_finished:
break
# Generation completed - prepare the output
if self.use_gpu:
outputs_np = np.array(input_ids[0].to('cpu'))
else:
outputs_np = np.array(input_ids[0])
output_only = outputs_np[input_token_len:]
output_str = self.tokenizer.decode(output_only)
# post-processing clean-up - stop at endoftext
eot = output_str.find("<|endoftext|>")
if eot > -1:
output_str = output_str[:eot]
# new post-processing clean-up - stop at </s>
eots = output_str.find("</s>")
if eots > -1:
output_str = output_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = output_str.find("<bot>:")
if bot > -1:
output_str = output_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = output_str.find("<s>")
if boss > -1:
output_str = output_str[boss + len("<s>"):]
# end - post-processing
total_len = len(outputs_np)
usage = {"input": input_token_len,
"output": total_len - input_token_len,
"total": total_len,
"metric": "tokens",
"processing_time": time.time() - time_start}
if get_first_token_speed:
usage.update({"first_token_processing_time": first_token_processing_time})
output_response = {"llm_response": output_str, "usage": usage}
if self.get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output_str
self.usage = usage
self.final_prompt = text_prompt
self.register()
return output_response
def fc_prompt_engineer(self, context, params=None, function=None):
""" Prompt engineering for Function Call prompts. """
if not params:
params = self.primary_keys
# add safety check in looking for default self.function pulled from model card
if not function:
if self.function:
if isinstance(self.function,list):
if len(self.function) > 0:
function = self.function[0]
else:
function = self.function
# if not successful identifying a function, then choose 'classify' by default
if not function:
function = "classify"
# prepare SLIM prompt
class_str = ""
for key in params:
class_str += str(key) + ", "
if class_str.endswith(", "):
class_str = class_str[:-2]
f = str(function)
# key templating format for SLIM function calls
full_prompt = "<human>: " + context + "\n" + "<{}> {} </{}>".format(f, class_str, f) + "\n<bot>:"
full_prompt = full_prompt + self.trailing_space
return full_prompt
def register_top_logits(self, next_token_logit):
""" Retrieves the logits for current sample, and packages into indexed top list and
registers in self.logit_record. """
# assumes input of next_token_logit from generation script
# will be a tensor of shape [1,vocab_size]
logit_size = next_token_logit.shape[-1]
logit = torch.squeeze(next_token_logit)
if self.use_gpu:
logit_array = np.array(logit.to('cpu'))
else:
logit_array = np.array(logit)
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
# by default, self.top_logit_count = 10, will get the top 10 highest values in logit output
for x in range(0, self.top_logit_count):
# experiment - rounding the long float number
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1],3))
top_logits.append(pair)
self.logits_record.append(top_logits)
return top_logits
def function_call(self, context, function=None, params=None, get_logits=True,
temperature=-99, max_output=None):
""" This is the key inference method for SLIM models - takes a context passage and a key list
which is packaged in the prompt as the keys for the dictionary output"""
self.context = context
# only assign self.function if a function has been passed in the call
if function:
self.function = function
if not self.fc_supported:
logger.warning("HFGenerativeModel - loaded model does not support function calls. "
"Please either use the standard .inference method with this model, or use a "
"model that has 'function_calls' key set to True in its model card.")
return []
# reset and start from scratch with new function call
self.output_tokens = []
self.logits_record = []
if temperature != -99:
self.temperature = temperature
if max_output:
self.target_requested_output_tokens = max_output
if get_logits:
self.get_logits = get_logits
if params:
self.primary_keys = params
# call to preview (not implemented by default)
self.preview()
if not self.primary_keys:
logger.warning("warning: function call - no keys provided - function call may yield unpredictable results")
# START - route to api endpoint
if self.api_endpoint:
return self.function_call_over_api_endpoint(model_name=self.model_name,
context=self.context,params=self.primary_keys,
function=self.function,
api_key=self.api_key,get_logits=self.get_logits)
# END - route to api endpoint
prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=self.function)
# second - tokenize to get the input_ids
tokenizer_output = self.tokenizer.encode(prompt)
input_token_len = len(tokenizer_output)
input_ids = torch.tensor(tokenizer_output).unsqueeze(0)
# explicit check and setting to facilitate debugging
if self.use_gpu:
input_ids = input_ids.to('cuda')
else:
input_ids = input_ids.to('cpu')
# time start
time_start = time.time()
# This simplified greedy sampling generation loop was derived from and inspired by ideas in the
# HuggingFace transformers library generation class.
# https: //github.com/huggingface/transformers/tree/main/src/transformers/generation
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc.team, and NVIDIA Corporation.
# Licensed under the Apache License, Version 2.0 (the "License")
# default settings
pad_token_id = 0
# for most models, eos_token_id = 0, but llama and mistral = 2
eos_token_id = [self.eos_token_id]
# eos_token_id = [0]
eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device)
# keep track of which sequences are already finished
unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
this_peer_finished = False # used by synced_gpus only
# auto-regressive generation
new_tokens_generated = 0
attn_mask = torch.ones(input_ids.shape[1]).unsqueeze(0)
# explicit check and setting to facilitate debugging, if needed
if self.use_gpu:
attn_mask = attn_mask.to('cuda')
else:
attn_mask = attn_mask.to('cpu')
batch_size = input_ids.shape[0]
seq_len = input_ids.shape[1]
pkv = None
while True:
inp_one_time: torch.LongTensor = input_ids
if new_tokens_generated > 0:
inp_one_time = input_ids[:, -1:]
# explicit check and setting to facilitate debugging, if needed
if self.use_gpu:
inp0 = inp_one_time.to('cuda')
inp1 = attn_mask.to('cuda')
else:
inp0 = inp_one_time.to('cpu')
inp1 = attn_mask.to('cpu')
# inp3 = torch.LongTensor([new_tokens_generated])
# need to invoke forward pass on model
# outputs = self.model(inp0,inp1,pkv)
with torch.no_grad():
outputs = self.model(input_ids=inp0, attention_mask=inp1, past_key_values=pkv, return_dict=True)
new_tokens_generated += 1
next_token_logits = outputs.logits[:, -1, :]
# option to capture logits for analysis
# if self.get_logits: self.register_top_logits(next_token_logits)
if self.temperature and self.sample:
next_token_scores = next_token_logits / self.temperature
else:
next_token_scores = next_token_logits
# get token from logits
probs = torch.nn.functional.softmax(next_token_scores, dim=-1)
if not self.sample:
# will pull the 'top logit' only
next_tokens = torch.argmax(probs).unsqueeze(0)
else:
# will apply probabilistic sampling
next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
# option to capture logits and output tokens for analysis
if self.get_logits:
self.register_top_logits(next_token_logits)
# capture the output tokens
if self.use_gpu:
next_tokens_np = np.array(next_tokens.to('cpu'))
else:
next_tokens_np = np.array(next_tokens)
self.output_tokens.append(next_tokens_np[0])
# finished sentences should have their next token be a padding token
if eos_token_id is not None:
next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences)
# update generated ids, model inputs, and length for next step
input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
# testing output in progress starts here
"""
logger.debug(f"update: input_ids - {input_ids}")
# outputs_detached = outputs.to('cpu')
outputs_np = np.array(input_ids[0])
output_str = self.tokenizer.decode(outputs_np)
logger.debug(f"update: output string - {output_str}")
"""
# end - testing output in progress
pkv = outputs.past_key_values
# update attention mask
attn_mask = torch.cat([attn_mask, attn_mask.new_ones((attn_mask.shape[0], 1))], dim=-1)
# if eos_token was found in one sentence, set sentence to finished
if eos_token_id_tensor is not None:
unfinished_sequences = unfinished_sequences.mul(
next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(
dim=0)
)
# stop when each sentence is finished
if unfinished_sequences.max() == 0:
this_peer_finished = True
# stop if we exceed the maximum length
if new_tokens_generated >= self.target_requested_output_tokens:
this_peer_finished = True
if this_peer_finished:
break
# Generation completed - prepare the output
if self.use_gpu:
outputs_np = np.array(input_ids[0].to('cpu'))
else:
outputs_np = np.array(input_ids[0])
output_only = outputs_np[input_token_len:]
output_str = self.tokenizer.decode(output_only)
# post-processing clean-up - stop at endoftext
eot = output_str.find("<|endoftext|>")
if eot > -1:
output_str = output_str[:eot]
# new post-processing clean-up - stop at </s>
eots = output_str.find("</s>")
if eots > -1:
output_str = output_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = output_str.find("<bot>:")
if bot > -1:
output_str = output_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = output_str.find("<s>")
if boss > -1:
output_str = output_str[boss + len("<s>"):]
# end - post-processing
total_len = len(outputs_np)
usage = {"input": input_token_len,
"output": total_len - input_token_len,
"total": total_len,
"metric": "tokens",
"processing_time": time.time() - time_start}
try:
output_value = ast.literal_eval(output_str)
output_type = "dict"
# allow for multiple valid object types - will expand over time
if isinstance(output_value,dict): output_type = "dict"
if isinstance(output_value,list): output_type = "list"
usage.update({"type": output_type})
except:
# could not convert automatically to python object
output_type = "string"
usage.update({"type": output_type})
output_value = output_str
# INSERT NEW HERE
if self.auto_remediate_function_call_output:
# attempt to remediate
output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str)
usage.update({"type": output_type, "remediation": True})
output_value = output_rem
if output_type == "string":
logger.warning("update: automatic conversion of function call output failed, and attempt to "
"remediate was not successful - %s ", output_str)
else:
logger.info("update: function call output could not be automatically converted, but remediation "
"was successful to type - %s ", output_type)
# INSERT ENDS HERE
output_response = {"llm_response": output_value, "usage": usage}
if get_logits:
output_response.update({"logits": self.logits_record})
output_response.update({"output_tokens": self.output_tokens})
self.logits = self.logits_record
# output inference parameters
self.llm_response = output_value
self.usage = usage
self.final_prompt = prompt
self.register()
return output_response
def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False):
""" Called by .inference method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
self.prompt=prompt
self.context=context
# preview call before invoking inference over rest api
self.preview()
import ast
import requests
url = self.api_endpoint + "{}".format("/")
output_raw = requests.post(url, data={"model_name": self.model_name,
"question": self.prompt,
"context": self.context,
"api_key": self.api_key,
"max_output": self.max_output,
"temperature": self.temperature})
try:
output = json.loads(output_raw.text)
# will attempt to unpack logits - but catch any exceptions and skip
if "logits" in output:
try:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
except:
output["logits"] = []
# will attempt to unpack output tokens - but catch any exceptions and skip
if "output_tokens" in output:
try:
# ot_int = [int(x) for x in output["output_tokens"]]
# output["output_tokens"] = ot_int
output_tokens = ast.literal_eval(output["output_tokens"])
output["output_tokens"] = output_tokens
except:
output["output_tokens"] = []
except:
logger.warning("warning: api inference was not successful")
output = {"llm_response": "api-inference-error", "usage": {}}
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="",
function=None, endpoint_base=None, api_key=None, get_logits=False):
""" Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
self.context = context
self.tool_type = tool_type
self.model_name = model_name
# send to api agent server
import ast
import requests
if endpoint_base:
self.api_endpoint = endpoint_base
if api_key:
# e.g., "demo-test"
self.api_key = api_key
if not params:
self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type]
mc = ModelCatalog().lookup_model_card(self.model_name)
if "primary_keys" in mc:
params = mc["primary_keys"]
self.primary_keys = params
if function:
self.function = function
self.prompt = prompt
# preview before invoking rest api
self.preview()
url = self.api_endpoint + "{}".format("/agent")
output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key,
"tool_type": self.tool_type,
"function": self.function,
"params": self.primary_keys, "max_output": 50,
"temperature": 0.0, "sample": False, "prompt": self.prompt,
"context": self.context, "get_logits": True})
try:
# output = ast.literal_eval(output_raw.text)
output = json.loads(output_raw.text)
if "logits" in output:
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
if "output_tokens" in output:
ot_int = [int(x) for x in output["output_tokens"]]
output["output_tokens"] = ot_int
# need to clean up logits
except:
logger.warning("warning: api inference was not successful")
output = {}
logger.info(f"TEST: executed Agent call over API endpoint - {model_name} - {function} - {output}")
# output inference parameters
self.llm_response = output["llm_response"]
self.usage = output["usage"]
self.final_prompt = prompt
if "logits" in output:
self.logits = output["logits"]
if "output_tokens" in output:
self.output_tokens = output["output_tokens"]
self.register()
return output
class GGUFGenerativeModel(BaseModel):
""" Implementation of GGUF Model class - instantiate and run inferences and function calls using
GGUF llama.cpp models """
def __init__(self, model_name=None, model_card=None, api_key=None, prompt_wrapper=None, instruction_following=False,
context_window=2048, use_gpu_if_available=True, get_logits=False,
sample=True, max_output=100, temperature=0.3, api_endpoint=None, **kwargs):
super().__init__(**kwargs)
logger.debug("GGUFGenerativeModel - constructing GGUF model.")
self.model_class = "GGUFGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.prompt = None
self.final_prompt = None
# set verbose level in environ level - will be picked up by callback in llama_cpp
os.environ["llama_cpp_verbose"] = GGUFConfigs().get_config("llama_cpp_verbose")
# os.environ["llama_cpp_verbose"] = "ON"
# adding new parameters - use_sampling, temperature, max_output
self.use_sampling = sample
self.sample = sample
self.get_logits = get_logits
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.auto_remediate_function_call_output = True
# default safety check in GGUF Configs that can be adjusted
gguf_configs_max = GGUFConfigs().get_config("max_output_tokens")
if max_output > gguf_configs_max:
# truncate max output to GGUFConfigs max
# logger.warning(f"update: requested output len - {max_output} > {gguf_configs_max}, which is the "
# f"current GGUF default max.\n--Truncating to {gguf_configs_max} output tokens.\n--Note: "
# f"to change GGUF default max to new integer amount, say 500:\n "
# f" GGUFConfigs().set_config(\"max_output_tokens\", 500)"
# )
max_output = gguf_configs_max
self.max_output = max_output
self.n_seq_max = max_output
self.target_requested_output_tokens = self.n_seq_max
self.max_total_len = 2048
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
self.max_output_len = self.n_seq_max
self.model_name = model_name
self.prompt_wrapper = prompt_wrapper
self.instruction_following = instruction_following
self.trailing_space = ""
self.separator = "\n"
self.eos_token_id = 0
self.add_prompt_engineering = False
self.add_context = ""
self.model_type = "gguf"
self.model_card = model_card
self.gguf_file = None
self.gguf_repo = None
self.primary_keys = None
self.function = None
self.hf_tokenizer_name = None
self.fc_supported = False
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "tokenizer" in model_card:
self.hf_tokenizer_name = model_card["tokenizer"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
else:
self.trailing_space = ""
if "eos_token_id" in model_card:
self.eos_token_id = model_card["eos_token_id"]
if "context_window" in model_card:
self.max_total_len = model_card["context_window"]
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
if "gguf_file" in model_card:
self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf"
if "gguf_repo" in model_card:
self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
# temperature configuration
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at GGUFConfigs default
self.temperature = GGUFConfigs().get_config("temperature_default")
# gguf specific attributes
self._lib = None
self._model = None
self._ctx = None
self._batch = None
self.model_path = None
self.model_params = None
self.context_params = None
# new option to 'force' use of cuda lib, and over-ride safety checks
if GGUFConfigs().get_config("force_gpu"):
self.use_gpu = True
else:
if sys.platform.lower() not in GGUFConfigs().get_config("cuda_platforms"):
self.use_gpu = False
else:
# min drivers set to the lowest level for CUDA 12.1 on Linux
min_drivers = [525, 60]
if sys.platform.lower() == "win32":
min_drivers = GGUFConfigs().get_config("cuda_windows_driver_min")
gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers)
# use_gpu set to TRUE only if:
# (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS
# (2) use_gpu set to True in GGUFConfigs
# (3) use_gpu_if_available flag set to True (by default)
# (4) cuda found and drivers current via direct polling of nvidia-smi executable in
# ModelCatalog.gpu_available method
self.use_gpu = (GGUFConfigs().get_config("use_gpu")
and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms")
and gpu_available["drivers_current"] and gpu_available["gpu_found"]
and use_gpu_if_available)
# set default minimum
self.n_batch = 2048
# self.n_batch = 512
self.last_n_tokens_size = 64
# by default
self._logits_all = False
self._n_vocab = None
self._n_ctx = None
self._token_nl = None
self._token_eos = None
self._candidates = None
self.input_ids = None
self.scores = None
self.n_tokens = 0
self.prev = []
self.grammar = None
for key, value in GGUFConfigs().get_sampling_params().items():
setattr(self, key, value)
# no api key expected or required
self.api_key = api_key
self.api_endpoint = api_endpoint
self.error_message = "\nUnable to identify and load GGUF Generative model."
self.prompt = ""
self.context = ""
self.tool_type = None
self.model_repo_path = None
self._sampler = None
self.vocab = None
self.input_token_count = 0
self.output_token_count = 0
self.post_init()
def load_model_for_inference(self, model_repo_path, model_card=None, **kwargs):
""" Loads and instantiates model along with other required objects. """
self.model_repo_path = model_repo_path
if model_card:
self.model_card = model_card
# validate before loading
self.validate()
# load shared library
self._lib = self._load_llama_cpp_shared_library()
self._lib = add_ctypes_declarations(self._lib)
if not GGUFConfigs().get_config("backend_initialized"):
# is this backend init required?
self._lib.llama_backend_init()
GGUFConfigs().set_config("backend_initialized", True)
self._lib.llama_log_set(llama_log_callback, ctypes.c_void_p(0))
self.model_params = self._lib.llama_model_default_params()
# update model params parameters
# important to set this correctly for Mac performance
self.model_params.n_gpu_layers = 50
# deprecated - change default split_mode from 1 -> 0
# self.model_params.split_mode = 0
self.model_params.main_gpu = 0
self.model_params.vocab_only = False
self.model_params.use_mmap = True
self.model_params.use_mlock = False
if self.use_gpu:
# on darwin, keep at 0 - on win32 and linux - set to 50 by default (e.g., shift all model layers to GPU)
if sys.platform.lower() == "win32" or sys.platform.lower().startswith("linux"):
self.model_params.n_gpu_layers = GGUFConfigs().get_config("n_gpu_layers")
# update context parameters
self.context_params = self._lib.llama_context_default_params()
# sets minimum of 2048, but will extend if context_window is larger (e.g., 4096/8192+)
self.context_params.n_ctx = max(2048, self.max_total_len)
self.context_params.n_batch = self.n_batch
n_ubatch = 512
self.context_params.n_ubatch = min(self.n_batch, n_ubatch)
# check on QC/ARM if 6 & 12 are ideal
# big improvement on MAC with formula below
# QC/ARM = 6
import multiprocessing
self.context_params.n_threads = max(multiprocessing.cpu_count() // 2, 1)
# QC/ARM = 12
self.context_params.n_threads_batch = multiprocessing.cpu_count()
self.context_params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED
self.context_params.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED
self.context_params.rope_freq_base = 0.0 # (rope_freq_base if rope_freq_base != 0.0 else 0)
self.context_params.rope_freq_scale = 0.0
# changed: defaults changed in llama cpp from build b6323 -> b6325
# self.context_params.yarn_ext_factor = -1.0
# self.context_params.yarn_attn_factor = 1.0
# self.context_params.yarn_beta_fast = 32.0
# self.context_params.yarn_beta_slow = 1.0
# end changes
self.context_params.type_k = 1
self.context_params.type_v = 1
self.context_params.offload_kqv = True
self.context_params.yarn_orig_ctx = 0
self.context_params.no_perf = False
# changes - llama cpp change from b6323 -> b6325
self.context_params.flash_attn = 0 # False
# self.context_params.flash_attn_type = 0
# end changes
self.context_params.embedding = False
self.context_params.swa_full = None
self.context_params.op_offloat = None
self.context_params.kv_unified = False
if model_card:
self.model_name = model_card["model_name"].split("/")[-1]
self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf",
self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
self.model_path = os.path.join(model_repo_path, self.gguf_file)
# loads and instantiates the key objects
self._model = _LlamaModel(self._lib, path_model=self.model_path, params=self.model_params)
self._ctx = _LlamaContext(self._lib, model=self._model, params=self.context_params)
self._batch = _LlamaBatch(self._lib, n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx)
self.vocab = self._lib.llama_model_get_vocab(self._model.model)
self._n_vocab = self.n_vocab()
self._n_ctx = self.n_ctx()
self._token_nl = self.token_nl()
self._token_eos = self.token_eos()
self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab)
self.input_ids = np.ndarray((self._n_ctx,), dtype=np.intc)
self.scores = np.ndarray((self._n_ctx, self._n_vocab), dtype=np.single)
self._sampler = self._init_sampler()
logger.info("GGUFGenerativeModel - loaded model - ready for inference")
return self
def _load_llama_cpp_shared_library(self):
""" Loads llama_cpp shared library - checks if a custom lib path has been configured - otherwise,
it loads the llmware provided dynamic libraries based on the platform/system. """
# check first if custom_lib_path - expected to be full path to custom so/dylib file
custom_path = GGUFConfigs().get_config("custom_lib_path")
cdll_args = dict()
# add option to fall_back if CUDA driver can not be loaded correctly to CPU driver for that OS
fall_back_option = ""
if custom_path:
if os.path.exists(custom_path):
_lib_paths = [custom_path]
else:
raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib")
else:
_base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf")
_lib_paths = []
system_platform = sys.platform.lower()
# Determine the file extension based on the platform
if system_platform.startswith("linux"):
# three linux versions supported - linux_x86 and linux_cuda
machine = os.uname().machine.lower()
if machine == "aarch64" and self.use_gpu:
_lib_paths.append(os.path.join(_base_path,
GGUFConfigs().get_config("linux_aarch64_cuda_lib"),
GGUFConfigs().get_config("linux_cuda")))
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"),
GGUFConfigs().get_config("linux_cuda")))
# will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86"))
else:
# by default load the cpu x86 lib
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86")))
elif system_platform == "darwin":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"),
GGUFConfigs().get_config("mac_metal")))
elif sys.platform == "win32":
import platform
if platform.machine().lower() == "arm64":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"),
GGUFConfigs().get_config("windows_arm64")))
# windows cuda
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"),
GGUFConfigs().get_config("windows_cuda")))
# new - will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows"))
else:
# main case - windows x86
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows")))
else:
raise LLMWareException(message=f"No matching llama.cpp binary for platform - {system_platform}")
# Add the library directory to the DLL search path on Windows (if needed)
if sys.platform == "win32" and sys.version_info >= (3, 8):
os.add_dll_directory(str(_base_path))
# need to review
if "CUDA_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths:
logger.debug(f"Loading llama cpp backend - {_lib_path}")
if not os.path.exists(_lib_path):
if fall_back_option:
_lib_path = fall_back_option
if os.path.exists(_lib_path):
try:
return ctypes.cdll.LoadLibrary(str(_lib_path))
except Exception as e:
# if fail, and CUDA selected, then try to fall back to matching CPU version
if fall_back_option:
try:
logger.warning("Not successful loading preferred lib so reverting to fallback lib.")
return ctypes.cdll.LoadLibrary(str(_lib_path))
except:
# if fall-back fails
raise GGUFLibNotLoadedException("llama_cpp_backend",
sys.platform.lower(),
self.use_gpu,
_lib_path,
custom_path)
else:
raise GGUFLibNotLoadedException("llama_cpp_backend" ,sys.platform.lower(),
self.use_gpu, _lib_path, custom_path)
# if not loaded
raise LLMWareException(message=f"GGUFGenerativeModel - attempting to load llama cpp backend lib - "
f"Llama cpp backend not found.")
def _init_sampler(self):
# create sampler
# default params are struct
params = llama_sampler_chain_params()
self._sampler = self._lib.llama_sampler_chain_init(params)
temp = 0.0
if temp < 0.0:
# sampler.add_softmax()
self._lib.llama_sampler_chain_add(self._sampler, self._lib.llama_sampler_init_softmax())
# sampler.add_dist(self._seed)
elif temp == 0.0:
# sampler.add_greedy()
greedy_sampler = self._lib.llama_sampler_init_greedy()
self._lib.llama_sampler_chain_add(self._sampler, greedy_sampler)
return self._sampler
def sample_gguf(self, idx=None):
""" Adapted to sample_gguf to avoid potential name space conflicts. """
# assert self.n_tokens > 0
tmp_sampler = False
if self._sampler is None:
tmp_sampler = True
self._sampler = self._init_sampler()
ridx = idx - self.n_tokens if idx is not None else -1
assert self.ctx is not None
token = self._lib.llama_sampler_sample(self._sampler, self._ctx.ctx, ridx)
# token = int(self.logits_record[-1][0][0])
if tmp_sampler:
self._sampler = None
return token
def _inference(self, prompt):
""" Tokenizes the prompt and executes generation loop. """
t0 = time.time()
completion_tokens = [] if len(prompt) > 0 else [self.token_bos()]
prompt_tokens = (
(
self.tokenize(prompt.encode("utf-8"), special=True)
if prompt != ""
else [self.token_bos()]
)
if isinstance(prompt, str)
else prompt
)
# confirm that input is smaller than context_window
input_len = len(prompt_tokens)
context_window = self.n_ctx()
if input_len > context_window:
logger.info("GGUFGenerativeModel - input is too long for model context window - truncating")
min_output_len = 10
prompt_tokens = prompt_tokens[0:context_window - min_output_len]
input_len = len(prompt_tokens)
text = b""
# first token capture starts here
get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed")
token_counter = 0
t_gen_start = time.time()
first_token_processing_time = -1.0
for token in self.generate(prompt_tokens):
# first token capture
if get_first_token_speed:
if token_counter == 0:
first_token_processing_time = time.time() - t_gen_start
token_counter += 1
# first token capture ends here
if self.get_logits:
self.register_top_logits()
self.output_tokens.append(token)
if token == self._token_eos:
text = self.detokenize(completion_tokens)
break
completion_tokens.append(token)
# stop at max output len
if len(completion_tokens) >= self.max_output_len:
text = self.detokenize(completion_tokens)
break
# stop if combined input + output at context window size
if (input_len + len(completion_tokens)) >= context_window:
text = self.detokenize(completion_tokens)
break
text_str = text.decode("utf-8", errors="ignore")
# post-processing clean-up - stop at endoftext
eot = text_str.find("<|endoftext|>")
if eot > -1:
text_str = text_str[:eot]
# new post-processing clean-up - stop at </s>
eots = text_str.find("</s>")
if eots > -1:
text_str = text_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = text_str.find("<bot>:")
if bot > -1:
text_str = text_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = text_str.find("<s>")
if boss > -1:
text_str = text_str[boss + len("<s>"):]
# end - post-processing
if get_first_token_speed:
output = {"llm_response": text_str,
"usage": {"input": len(prompt_tokens), "output": len(completion_tokens),
"total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens",
"processing_time": time.time() - t0,
"first_token_processing_time": first_token_processing_time}}
else:
output = {"llm_response": text_str,
"usage": {"input": len(prompt_tokens), "output": len(completion_tokens),
"total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens",
"processing_time": time.time() - t0}}
if self.get_logits:
output.update({"logits": self.logits_record})
output.update({"output_tokens": self.output_tokens})
return output
def generate(self, tokens, reset=True):
""" Generator that samples the model and yields tokens until stopped. """
logger.debug("GGUFGenerativeModel - starting generation loop")
# Reset the model state
if reset:
self.reset()
sample_idx = self.n_tokens + len(tokens) - 1
tokens = list(tokens)
tokens_created = 0
input_start_len = len(tokens)
memory = self._ctx.memory
# Eval and sample
while True:
self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1)
for i in range(0, len(tokens), self.n_batch):
batch = tokens[i: min(len(tokens), i + self.n_batch)]
n_past = self.n_tokens
n_tokens = len(batch)
self._batch.set_batch(batch=batch, n_past=n_past, logits_all=self._logits_all)
return_code = self._lib.llama_decode(self._ctx.ctx, self._batch.batch)
# TODO: add better error handling if return_code 1 - usually overflow of ctx
if return_code != 0:
raise RuntimeError(f"GGUFGenerativeModel - generate - llama_decode call returned {return_code} - in most cases, this "
f"is due to exceeding the maximum context window.")
self.input_ids[n_past: n_past + n_tokens] = batch
rows = n_tokens
cols = self._n_vocab
offset = (0 if self._logits_all else n_tokens - 1)
if self._logits_all:
rows = n_tokens
cols = self._n_vocab
logits = np.ctypeslib.as_array(
self._ctx.get_logits(), shape=(rows * cols,))
self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits
self.n_tokens += n_tokens
# TODO: inserting test for logits
# self.register_top_logits()
while sample_idx < self.n_tokens:
logits = self._scores[-1, :]
self.prev = list(self.eval_tokens)
# note: call to .sample_gguf method
token = self.sample_gguf(idx=sample_idx) # (logits_array=logits)
self.accept(id=id, apply_grammar=None)
tokens_created += 1
sample_idx += 1
tokens_or_none = yield token
tokens.clear()
tokens.append(token)
if tokens_or_none is not None:
tokens.extend(tokens_or_none)
if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]:
self.n_tokens = sample_idx
self._lib.llama_memory_seq_rm(self._lib.llama_get_memory(self._ctx.ctx), -1, self.n_tokens, -1)
# self._lib.llama_kv_cache_seq_rm(self._ctx.ctx, -1, self.n_tokens, -1)
break
if tokens_created > self.max_output_len:
logger.info("GGUFGenerativeModel - stopping generation loop - reached limit of "
"max output len")
break
def tokenize(self, text, add_bos=True, special=False):
""" Tokenizes text. """
n_ctx = self.n_ctx_train()
tokens = (ctypes.c_int32 * n_ctx)()
# change from self._model.model
n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_ctx, add_bos, special)
if n_tokens < 0:
n_tokens = abs(n_tokens)
tokens = (ctypes.c_int32 * n_tokens)()
n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_tokens, add_bos, special)
if n_tokens < 0:
raise RuntimeError(f"GGUFGenerativeModel - tokenization error - {text} - "
f"n_tokens={n_tokens}")
return list(tokens[:n_tokens])
def detokenize(self, tokens, special: bool = False) -> bytes:
output = b""
size = 32
buffer = (ctypes.c_char * size)()
for token in tokens:
n = self._lib.llama_token_to_piece(
# replace: self.model
self.vocab, llama_token(token), buffer, size, 0, special
)
assert n <= size
output += bytes(buffer[:n])
# NOTE: Llama1 models automatically added a space at the start of the prompt
# this line removes a leading space if the first token is a beginning of sentence token
return (
output[1:]
if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" "
else output
)
def accept(self, id, apply_grammar):
""" Formal step post sampling that 'accepts' and adds the token id to the running generation. """
if apply_grammar and self.grammar is not None:
self._lib.llama_grammar_accept_token(self._ctx.ctx, self.grammar.grammar, id)
self.prev.append(id)
def register_top_logits(self):
""" Gets the top logits and keeps a running log for output analysis. """
# TODO: there is issue with first logit computation - not corresponding to first token
logit_pointer = self._lib.llama_get_logits(self._ctx.ctx)
logit_size = self.n_vocab()
logit_array = np.zeros(logit_size)
for x in range(0, logit_size):
logit_array[x] = logit_pointer[x]
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
for x in range(0, self.top_logit_count):
# experiment - try rounding the float number
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3))
top_logits.append(pair)
self.logits_record.append(top_logits)
return top_logits
def set_api_key(self, api_key, env_var="USER_MANAGED_GGUF_API_KEY"):
""" Sets API key - generally not used in GGUF models. """
# set api_key
os.environ[env_var] = api_key
logger.info("GGUFGenerativeModel - added and stored GGUF api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_GGUF_API_KEY"):
""" Gets API key - generally not used in GGUF models. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.error("GGUFGenerativeModel - _get_api_key could not successfully retrieve value from: %s ", env_var)
return self.api_key
def token_counter(self, text_sample):
if not text_sample:
tokens = 0
else:
tokens = len(self.tokenize(text_sample.encode("utf-8")))
return tokens
@property
def ctx(self):
return self._ctx.ctx
@property
def model(self):
return self._model.model
@property
def _input_ids(self):
return self.input_ids[: self.n_tokens]
@property
def _scores(self):
return self.scores[: self.n_tokens, :]
@property
def eval_tokens(self):
return deque(self.input_ids[: self.n_tokens].tolist(), maxlen=self._n_ctx)
@property
def eval_logits(self):
return deque(
self.scores[: self.n_tokens, :].tolist(),
maxlen=self._n_ctx if self._logits_all else 1,
)
def reset(self):
self.n_tokens = 0
def n_ctx(self):
return self._lib.llama_n_ctx(self._ctx.ctx)
def n_ctx_train(self):
return self._lib.llama_n_ctx_train(self._model.model)
def n_vocab(self):
n_vocab = self._lib.llama_n_vocab(self._lib.llama_model_get_vocab(self._model.model))
return n_vocab
def token_eos(self):
eos = self._lib.llama_token_eos(self.vocab)
return eos
def token_bos(self):
bos = self._lib.llama_token_bos(self.vocab)
return bos
def token_nl(self):
token_nl = self._lib.llama_token_nl(self._lib.llama_model_get_vocab(self._model.model))
return token_nl
def unload_model(self):
""" Unloads a model to release memory """
# note: removing pointer seems to safely remove from Python reference tracking
# --will evaluate under multiple scenarios if free explicitly needs to be called in llama.cpp engine
self._batch = None
self._ctx = None
self._model = None
return 0
def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, inference_dict=None,
get_logits=False):
""" Main method for inference generation. """
self.prompt = prompt
# first prepare the prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# update default handling for no add_prompt_engineering
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# end - update
# show warning if function calling model
if self.fc_supported:
logger.info("GGUFGenerativeModel - this is a function calling model - using .inference may lead to unexpected "
"results. Recommended to use the .function_call method to ensure correct prompt "
"template packaging.")
# start with clean logits_record and output_tokens for each function call
self.logits_record = []
self.output_tokens = []
if get_logits:
self.get_logits = get_logits
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# preview before initiating inference over api
self.preview()
# START - route to api endpoint
if self.api_endpoint:
sd = self.to_state_dict()
return self.inference_over_api_endpoint(self.prompt, context=self.add_context,
inference_dict=inference_dict)
# END - route to api endpoint
text_prompt = self.prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# text_prompt = prompt_final + "\n"
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
text_prompt = prompt_final + self.trailing_space
output_response = self._inference(text_prompt)
# update linked to BaseModel
self.prompt = prompt
self.final_prompt = text_prompt
self.usage = output_response["usage"]
self.llm_response = output_response["llm_response"]
if "logits" in output_response:
self.logits = output_response["logits"]
self.register()
# end - update
return output_response
def function_call(self, context, function=None, params=None, get_logits=True,
temperature=-99.0, max_output=None):
""" This is the key inference method for SLIM models - takes a context passage and a key list
which is packaged in the prompt as the keys for python dictionary output"""
if not self.fc_supported:
logger.warning("GGUFGenerativeModel - loaded model does not support function calls. "
"Please either use the standard .inference method with this model, or use a GGUF "
"model that has 'function_calls' key set to True in its model card.")
return []
self.context = context
# start with clean logits_record and output_tokens for each function call
self.logits_record = []
self.output_tokens = []
if get_logits:
self.get_logits = get_logits
if params:
self.primary_keys = params
if not self.primary_keys:
logger.warning("GGUFGenerativeModel - function call - no keys provided - "
"function call may yield unpredictable results")
if not params:
params = self.primary_keys
if not function:
# pull from model card
if self.function:
if isinstance(self.function, list):
if len(self.function) > 0:
function = self.function[0]
else:
function = self.function
if not function:
function = "classify"
self.primary_keys = params
self.function = function
# preview before initiating api call
self.preview()
# START - route to api endpoint
if self.api_endpoint:
return self.function_call_over_api_endpoint(model_name=self.model_name,
context=self.context,params=self.primary_keys,
function=self.function,
api_key=self.api_key,get_logits=self.get_logits)
# END - route to api endpoint
# prepare SLIM prompt
class_str = ""
for key in params:
class_str += str(key) + ", "
if class_str.endswith(", "):
class_str = class_str[:-2]
f = str(self.function)
full_prompt = "<human>: " + self.context + "\n" + "<{}> {} </{}>".format(f, class_str, f) + "\n<bot>:"
full_prompt = full_prompt + self.trailing_space
text_prompt = full_prompt
if temperature != -99:
self.temperature = temperature
if max_output:
self.max_output_len = max_output
# call inference here
output_response = self._inference(text_prompt)
output_str = output_response["llm_response"]
try:
import ast
output_dict = ast.literal_eval(output_str)
output_type = "dict"
if isinstance(output_dict, dict): output_type = "dict"
if isinstance(output_dict, list): output_type = "list"
output_response["usage"].update({"type": output_type})
output_response.update({"llm_response": output_dict})
except:
output_type = "string"
output_response["usage"].update({"type": output_type})
if self.auto_remediate_function_call_output:
# attempt to automatically remediate
output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str)
if output_type != "string":
output_response["usage"].update({"type": output_type, "remediation": True})
output_response.update({"llm_response": output_rem})
if output_type == "string":
logger.warning("GGUFGenerativeModel - function call - automatic conversion of function call output failed, and attempt to "
"remediate was not successful - %s ", output_str)
else:
logger.info("GGUFGenerativeModel - function call output could not be automatically converted, but remediation "
"was successful to type - %s ", output_type)
# update linked to BaseModel
self.prompt = ""
self.final_prompt = full_prompt
self.usage = output_response["usage"]
self.llm_response = output_response["llm_response"]
if "logits" in output_response:
self.logits = output_response["logits"]
self.register()
# end - update
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, inference_dict=None,
get_logits=False, disable_eos=False, skip_pe_override=False):
""" Main method for text streaming generation. Returns a generator function that yields one
token at a time for real-time streaming to console or UI. """
# first prepare the prompt
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# update default handling for no add_prompt_engineering
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# show warning if function calling model
if self.fc_supported:
logger.info("GGUFGenerativeModel - this is a function calling model - using .inference may lead to unexpected "
"results. Recommended to use the .function_call method to ensure correct prompt "
"template packaging.")
# start with clean logits_record and output_tokens for each function call
self.logits_record = []
self.output_tokens = []
if get_logits:
self.get_logits = get_logits
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# preview before generation
self.preview()
if self.add_prompt_engineering and not skip_pe_override:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
prompt = prompt_final + self.trailing_space
if self.api_endpoint:
""" Not implemented """
# continue with local execution ...
# starts _inference here
completion_tokens = [] if len(prompt) > 0 else [self.token_bos()]
logger.info(f"GGUFGenerative - stream - model name - {self.model_name}")
prompt_tokens = (
(
self.tokenize(prompt.encode("utf-8"), special=True)
if prompt != ""
else [self.token_bos()]
)
if isinstance(prompt, str)
else prompt
)
# confirm that input is smaller than context_window
input_len = len(prompt_tokens)
context_window = self.n_ctx()
logger.info(f"GGUFGenerativeModel stream - input token len - {input_len}")
if input_len > context_window:
logger.warning("GGUFGenerativeModel - input is too long for model context window - truncating")
min_output_len = 10
prompt_tokens = prompt_tokens[0:context_window - min_output_len]
input_len = len(prompt_tokens)
text = b""
# disable_eos = True
token_list = []
for token in self.generate(prompt_tokens):
completion_tokens.append(token)
if not disable_eos:
if token == self._token_eos:
break
if len(completion_tokens) > self.max_output_len:
break
# stop if combined input + output at context window size
if (input_len + len(completion_tokens)) >= context_window:
break
new_token = self.detokenize([token]).decode('utf-8', errors='ignore')
# a little cleanup of 'think' tokens
if new_token == "<think>":
new_token = "<|think|>"
logger.info(f"GGUFGenerativeModel - stream - changing token to markdown safe - {new_token}")
if new_token == "</think>":
new_token = "<|endthink|>"
yield new_token
text_str = text.decode("utf-8", errors="ignore")
# turned off
self.register()
return text_str
def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="",
function=None, endpoint_base=None, api_key=None, get_logits=False):
""" Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
# send to api agent server
self.context = context
self.tool_type = tool_type
import ast
import requests
if endpoint_base:
self.api_endpoint = endpoint_base
if api_key:
# e.g., "demo-test"
self.api_key = api_key
if not params:
self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type]
mc = ModelCatalog().lookup_model_card(self.model_name)
if "primary_keys" in mc:
params = mc["primary_keys"]
if function:
self.function = function
self.prompt = prompt
self.primary_keys = params
# preview before invoking api
self.preview()
url = self.api_endpoint + "{}".format("/agent")
output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key,
"tool_type": self.tool_type,
"function": self.function, "params": self.primary_keys,
"max_output": 50,
"temperature": 0.0, "sample": False, "prompt": self.prompt,
"context": self.context, "get_logits": True})
try:
output = json.loads(output_raw.text)
# will attempt to unpack logits - but catch any exceptions and skip
if "logits" in output:
try:
import ast
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
except:
output["logits"] = []
# will attempt to unpack output tokens - but catch any exceptions and skip
if "output_tokens" in output:
try:
ot_int = [int(x) for x in output["output_tokens"]]
output["output_tokens"] = ot_int
# output_tokens = ast.literal_eval(output["output_tokens"])
# output["output_tokens"] = output_tokens
except:
output["output_tokens"] = []
# output = ast.literal_eval(output_raw.text)
except:
logger.warning("GGUFGenerativeModel - function_call_over_api_endpoint - api inference was not successful")
output = {"llm_response": "api-inference-error", "usage": {}}
# update linked to BaseModel
self.prompt = prompt
self.final_prompt = prompt
self.usage = output["usage"]
self.llm_response = output["llm_response"]
if "logits" in output:
self.logits = output["logits"]
self.register()
# end - update
return output
def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False):
""" Called by .inference method when there is an api_endpoint passed in the model constructor. Rather
than execute the inference locally, it will be sent over API to inference server. """
self.prompt = prompt
self.context = context
# preview before invoking inference over rest api
self.preview()
import ast
import requests
url = self.api_endpoint + "{}".format("/")
output_raw = requests.post(url, data={"model_name": self.model_name,
"question": self.prompt,
"context": self.context,
"api_key": self.api_key,
"max_output": self.max_output_len,
"temperature": self.temperature})
try:
output = json.loads(output_raw.text)
# will attempt to unpack logits - but catch any exceptions and skip
if "logits" in output:
try:
import ast
logits = ast.literal_eval(output["logits"])
output["logits"] = logits
except:
output["logits"] = []
# will attempt to unpack output tokens - but catch any exceptions and skip
if "output_tokens" in output:
try:
import ast
# ot_int = [int(x) for x in output["output_tokens"]]
# output["output_tokens"] = ot_int
output_tokens = ast.literal_eval(output["output_tokens"])
output["output_tokens"] = output_tokens
except:
output["output_tokens"] = []
except:
logger.warning("warning: api inference was not successful")
output = {"llm_response": "api-inference-error", "usage": {}}
# update linked to BaseModel
self.prompt = prompt
self.final_prompt = prompt
self.usage = output["usage"]
self.llm_response = output["llm_response"]
if "logits" in output:
self.logits = output["logits"]
self.register()
# end - update
return output
class WhisperCPPModel(BaseModel):
""" WhisperCPPModel is an implementation of the Whisper voice transcription model running on GGML, rather
than Pytorch. """
def __init__(self, model_name=None, model_card=None, use_gpu_if_available=True, **kwargs):
super().__init__(**kwargs)
self.model_class = "WhisperCPPModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.prompt = None
self.final_prompt = None
# set verbose level in environ level - will be picked up by callback in whisper_cpp
os.environ["whisper_cpp_verbose"] = GGUFConfigs().get_config("whisper_cpp_verbose")
self.WHISPER_SR = GGUFConfigs().get_config("whisper_sr")
self.strategy = GGUFConfigs().get_config("whisper_strategy")
self.n_threads = GGUFConfigs().get_config("whisper_threads")
self.language = GGUFConfigs().get_config("whisper_language")
self.format = GGUFConfigs().get_config("whisper_output_format")
self.tiny_diarize = GGUFConfigs().get_config("whisper_tiny_diarize")
self.beam_size = GGUFConfigs().get_config("whisper_beam_size")
self.greedy_best_of = GGUFConfigs().get_config("whisper_greedy_best_of")
self.temperature_inc = GGUFConfigs().get_config("whisper_temperature_inc")
self.remove_segment_markers = GGUFConfigs().get_config("whisper_remove_segment_markers")
self.model_card = model_card
self.model_name = model_name
self._lib = None
self.model_path = None
self.context = None
self.params = None
self.temperature = 0.0
self.duration = 0
self.translate = False
if sys.platform.lower() == "darwin":
self.whisper_use_legacy_mac = GGUFConfigs().get_config("whisper_use_legacy_mac")
else:
self.whisper_use_legacy_mac = False
# new option to 'force' use of cuda lib, and over-ride safety checks
if GGUFConfigs().get_config("force_gpu"):
self.use_gpu = True
else:
if not sys.platform.lower().startswith("linux"):
self.use_gpu = False
else:
# min drivers set to the lowest level for CUDA 12.1 on Linux
min_drivers = [525,60]
gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers)
# use_gpu set to TRUE only if:
# (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS
# (2) use_gpu set to True in GGUFConfigs
# (3) use_gpu_if_available flag set to True (by default)
# (4) cuda found and drivers current via direct polling of nvidia-smi executable in
# ModelCatalog.gpu_available method
self.use_gpu = (GGUFConfigs().get_config("use_gpu")
and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms")
and gpu_available["drivers_current"] and gpu_available["gpu_found"]
and use_gpu_if_available)
self.model_repo_path = None
self.post_init()
def load_model_for_inference(self, model_repo_path, model_card = None, **kwargs):
""" Loads and instantiates model along with other required objects. """
self.model_repo_path = model_repo_path
if model_card:
self.model_card = model_card
# validate before loading
self.validate()
# load shared library
self._lib = self._load_shared_library()
self._lib = self.add_ctypes_configs()
self._lib.whisper_log_set(whisper_log_callback, ctypes.c_void_p(0))
if model_card:
self.model_name = model_card["model_name"].split("/")[-1]
self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf",
self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
self.model_path = os.path.join(model_repo_path, self.gguf_file)
self.context = self._lib.whisper_init_from_file(self.model_path.encode('utf-8'))
self.params = self._lib.whisper_full_default_params(self.strategy)
self.params.n_threads = self.n_threads
# self.params.print_special = True
# self.params.print_progress = False
# set to True by default - will display in 'real-time' the transcription
# self.params.print_realtime = GGUFConfigs().get_config("whisper_cpp_realtime_display")
# self.params.print_timestamps = True
# self.params.tdrz_enable = self.tiny_diarize
# self.params.progress_callback = whisper_progress_callback(self.callback)
# self.params.temperature_inc = self.temperature_inc
# self.params.token_timestamps = True
# self.params.greedy.best_of = self.greedy_best_of
# self.params.beam_search.beam_size = self.beam_size
return self
def _load_shared_library(self):
""" Loads the libwhisper.cpp backend GGML engine that runs the model. """
# check first if custom_lib_path - expected to be full path to custom so/dylib file
custom_path = GGUFConfigs().get_config("whisper_cpp_lib_path")
fall_back_option = ""
cdll_args = dict()
if custom_path:
if os.path.exists(custom_path):
_lib_paths = [custom_path]
else:
raise LLMWareException(message=f"WhisperCPPModel - attempted to load whisper cpp backend lib - "
f"could not find path to custom lib - {custom_path}")
else:
# general case - will look for llama.cpp dynamic library included with llmware
_base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf")
_lib_paths = []
system_platform = sys.platform.lower()
# Determine the file extension based on the platform
if system_platform.startswith("linux"):
machine = os.uname().machine.lower()
if machine == "aarch64" and self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"),
GGUFConfigs().get_config("whisper_dgx")))
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"),
GGUFConfigs().get_config("whisper_linux_cuda")))
# new - will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("whisper_linux_x86"))
else:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("whisper_linux_x86")))
elif system_platform == "darwin":
if not self.whisper_use_legacy_mac:
mac_lib = GGUFConfigs().get_config("whisper_mac_metal")
else:
mac_lib = GGUFConfigs().get_config("whisper_mac_metal_legacy")
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"),mac_lib))
elif sys.platform == "win32":
import platform
if platform.machine().lower() == "arm64":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"),
GGUFConfigs().get_config("whisper_windows_arm64")))
# windows cuda
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"),
GGUFConfigs().get_config("whisper_windows")))
# new - will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("whisper_windows"))
else:
# main case - windows x86
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("whisper_windows")))
# Add the library directory to the DLL search path on Windows (if needed)
# if sys.platform == "win32" and sys.version_info >= (3, 8): os.add_dll_directory(str(_base_path))
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths:
if not os.path.exists(_lib_path):
if fall_back_option:
_lib_path = fall_back_option
if os.path.exists(_lib_path):
try:
return ctypes.CDLL(str(_lib_path), **cdll_args)
except Exception as e:
# NEW INSERT - if fail, and CUDA selected, then try to fall back to matching CPU version
if fall_back_option:
try:
logger.warning("update: Not successful loading primary lib, so reverting to secondary "
"driver (which may be slower).")
return ctypes.CDLL(str(fall_back_option), **cdll_args)
except:
# if fall-back fails
raise GGUFLibNotLoadedException("whisper_cpp_backend",
sys.platform.lower(),
self.use_gpu,
_lib_path,
custom_path)
else:
raise GGUFLibNotLoadedException("whisper_cpp_backend",sys.platform.lower(),
self.use_gpu, _lib_path, custom_path)
else:
logger.warning(f"update: looking for WhisperCPP lib - path does not exist - {str(_lib_path)}")
# Try to load the shared library, handling potential errors
# *** something has gone wrong - could not find the lib files
raise FileNotFoundError(f"Exception: WhisperCPP Shared library not found at paths - {str(_lib_paths)}")
# new method starts here
def inference(self, prompt, inference_dict=None):
""" Inference on Whisper model takes a single input 'prompt' which is a string corresponding to a
full file path pointing to the voice file to be transcribed, e.g.,
`/home/ubuntu/voice_samples/sample.wav
"""
self.prompt=prompt
if inference_dict:
if "translate" in inference_dict:
self.translate=inference_dict["translate"]
if "remove_segment_markers" in inference_dict:
self.remove_segment_markers = inference_dict["remove_segment_markers"]
# preview before starting inference
# self.preview()
# note: updated dependencies for improved efficiency
# previously, used librosa library
# replaced librosa with two librosa sub-dependencies that do most of the work
# e.g., soundfile, and soxr which results in smaller footprint for deployment
file = prompt
if not file.endswith(".wav"):
logger.info("update: WhisperCPPModel - inference - input file needs to be converted to .wav - "
"will try to do right now.")
new_file_path = Utilities().convert_media_file_to_wav(self.prompt,
save_path=LLMWareConfig().get_tmp_path(),
file_out="converted_file_tmp.wav")
if not new_file_path:
logger.warning("update: WhisperCPPModel - inference - conversion was not successful. "
"The most likely causes of this error - \n"
"1. File type is not supported - the following are the supported file types - "
"mp3, m4a, mp4, wma, aac, ogg, flv. \n"
"2. lib ffmpeg is not installed on your system. This is the core audio processing "
"library that handles the file conversion.\n"
"--to install on Mac: brew install ffmpeg \n"
"--to install on Linux: sudo apt install ffmpeg \n"
"--to install on Windows: see ffmpeg.org/download.html for download/install \n")
null_output = {"llm_response": "", "segments": []}
return null_output
else:
logger.info(f"update: WhisperCPPModel - inference - file conversion to .wav successful - "
f"new file at tmp path - {new_file_path}")
file = new_file_path
# loading new dependencies starts here
try:
import soundfile as sf
import soxr
except:
raise LLMWareException("WhisperCPPModel class requires dependencies of soundfile and soxr,"
"e.g., `pip install soundfile` and `pip install soxr`")
sfo = sf.SoundFile(file)
with sfo as sf_desc:
sr = sf_desc.samplerate
frame_duration = -1
data = sf_desc.read(frames=frame_duration, dtype=np.float32, always_2d=False).T
if self.WHISPER_SR != sr:
# y = resample(data, orig_sr=sr_native, target_sr=sr, res_type="soxr_hq")
ratio = float(sr) / self.WHISPER_SR
axis = -1
n_samples = int(np.ceil(data.shape[axis] * ratio))
yhat = np.apply_along_axis(soxr.resample, axis=axis, arr=data,
in_rate=sr, out_rate=self.WHISPER_SR, quality="soxr_hq")
data = np.asarray(yhat, dtype=np.float32)
# new dependencies end here
# replacing previous: data, sr = librosa.load(file, sr=self.WHISPER_SR)
try:
self.duration = float(data.shape[-1]) / self.WHISPER_SR
# self.duration = librosa.get_duration(y=data, sr=self.WHISPER_SR)
except:
self.duration = float(0.0)
data.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
self.params.language = self.language.encode('utf-8')
if prompt:
self.params.initial_prompt = prompt.encode('utf-8')
# self.params.temperature = self.temperature
# self.params.translate = self.translate
result = self._generate(data)
# output format options
output = result["text"]
if self.format == "srt":
output = '\n'.join([f'{i + 1}\n{self._format_time(s["start"])} --> '
f'{self._format_time(s["end"])}\n{s["text"]}\n'
for i, s in enumerate(result["segments"])])
if self.format == "vtt":
output = '\n'.join([f'{i + 1}\n{self._format_time(s["start"])} --> '
f'{self._format_time(s["end"])} align:middle\n{s["text"]}\n'
for i, s in enumerate(result["segments"])])
usage_dict = {"duration-seconds": self.duration, "segments": len(result["segments"]),
"language": self.language}
response = {"llm_response": output, "usage": usage_dict, "segments": result["segments"]}
# update linked to BaseModel
self.prompt = ""
self.final_prompt = ""
self.usage = response["usage"]
self.llm_response = response["llm_response"]
self.register()
# end - update
return response
def _generate(self, data):
""" Executes lib_whisper generation on data from audio file. """
w = self._lib.whisper_full(ctypes.c_void_p(self.context),
self.params,
data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
len(data))
if w != 0:
raise LLMWareException(message=f"Exception: WhisperCPPModel - inference: {w}")
segments = []
all_text = ""
text_chunks = []
n_segments = self._lib.whisper_full_n_segments(ctypes.c_void_p(self.context))
for i in range(n_segments):
t0 = self._lib.whisper_full_get_segment_t0(ctypes.c_void_p(self.context), i)/100.0
t1 = self._lib.whisper_full_get_segment_t1(ctypes.c_void_p(self.context), i)/100.0
txt = self._lib.whisper_full_get_segment_text(ctypes.c_void_p(self.context), i).decode('utf-8-sig',
errors='ignore')
if self.tiny_diarize:
# look for [_SOLM_] token to break segment - and will keep aggregating until found
if "[_SOLM_]" in txt:
txt += "\n\n"
if self.remove_segment_markers:
# removes leading [_BEG_] & trailing [_TT_XYZ] special tokens
txt_split = txt.split("[_TT_")[0]
txt_split = txt_split.strip()
if txt_split.startswith("[_BEG_]"):
txt_split= txt_split[len("[_BEG_]"):]
txt = " " + txt_split + " "
all_text += txt
text_chunks.append(txt)
n_tokens = self._lib.whisper_full_n_tokens(ctypes.c_void_p(self.context), i)
tokens = []
for j in range(n_tokens):
token_data = self._lib.whisper_full_get_token_data(ctypes.c_void_p(self.context), i, j)
tokens.append({
"id": token_data.id,
"prob": token_data.p,
"logprob": token_data.plog,
"pt": token_data.pt,
"pt_sum": token_data.ptsum,
})
segments.append({
"start": t0,
"end": t1,
"text": txt,
"tokens": tokens,
})
result = {"text": all_text.strip(), "text_chunks": text_chunks, "segments": segments}
return result
def __dealloc__(self):
# free the memory
self._lib.whisper_free(ctypes.c_void_p(self.context))
def unload_model(self):
self._lib = None
@staticmethod
def _format_time(t):
""" Helper utility that formats the time. """
msec = t * 10
hr = msec / (1000 * 60 * 60)
msec = msec - hr * (1000 * 60 * 60)
minu = msec / (1000 * 60)
msec = msec - minu * (1000 * 60)
sec = msec / 1000
msec = msec - sec * 1000
return f'{int(hr):02}:{int(minu):02}:{int(sec):02}.{int(msec):03}'
def abort_call_back(self, data):
do_nothing = 0
def callback(self, ctx, state, i, p):
do_nothing = 0
def add_ctypes_configs(self):
self._lib.whisper_init_from_file.argtypes = [ctypes.c_char_p]
self._lib.whisper_init_from_file.restype = ctypes.c_void_p
self._lib.whisper_full_default_params.argtypes = [ctypes.c_int]
if not self.whisper_use_legacy_mac:
self._lib.whisper_full_default_params.restype = whisper_full_params
else:
self._lib.whisper_full_default_params.restype = whisper_full_params_legacy
if not self.whisper_use_legacy_mac:
self._lib.whisper_full.argtypes = [ctypes.c_void_p, whisper_full_params, ctypes.POINTER(ctypes.c_float), ctypes.c_int]
else:
self._lib.whisper_full.argtypes = [ctypes.c_void_p, whisper_full_params_legacy, ctypes.POINTER(ctypes.c_float), ctypes.c_int]
self._lib.whisper_full.restype = ctypes.c_int
self._lib.whisper_full_n_segments.argtypes = [ctypes.c_void_p]
self._lib.whisper_full_n_segments.restype = ctypes.c_int
self._lib.whisper_full_get_segment_t0.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_t0.restype = ctypes.c_int64
self._lib.whisper_full_get_segment_t1.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_t1.restype = ctypes.c_int64
self._lib.whisper_full_get_segment_text.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_text.restype = ctypes.c_char_p
self._lib.whisper_full_n_tokens.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_n_tokens.restype = ctypes.c_int
self._lib.whisper_full_get_segment_t0.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_t0.restype = ctypes.c_int64
self._lib.whisper_full_get_segment_t1.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_t1.restype = ctypes.c_int64
self._lib.whisper_full_get_token_data.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
self._lib.whisper_full_get_token_data.restype = whisper_token_data
self._lib.whisper_full_n_segments.argtypes = [ctypes.c_void_p]
self._lib.whisper_full_n_segments.restype = ctypes.c_int
self._lib.whisper_full_get_segment_text.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_get_segment_text.restype = ctypes.c_char_p
self._lib.whisper_full_n_tokens.argtypes = [ctypes.c_void_p, ctypes.c_int]
self._lib.whisper_full_n_tokens.restype = ctypes.c_int
self._lib.whisper_full_get_token_data.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
self._lib.whisper_full_get_token_data.restype = whisper_token_data
self._lib.whisper_free.argtypes = [ctypes.c_void_p]
self._lib.whisper_free.restype = None
self._lib.whisper_log_set.artypes = [ctypes.c_void_p, ctypes.c_void_p]
self._lib.whisper_log_set.restype = None
return self._lib
class LLMWareSemanticModel(BaseModel):
""" LLMWareSemanticModel class implements the LLMWareSemanticModel API, which is based on the SentenceTransformer
architecture. """
def __init__(self, model_name=None, model=None, embedding_dims=None, max_len=150,
model_card=None, api_key=None, **kwargs):
super().__init__(**kwargs)
self.model_name = model_name
self.error_message = "\nUnable to process LLMWare Semantic Model. Please try again later"
self.max_input_len = 512
self.max_output_len = 512
self.max_len = max_len
# to be applied to 'passed-in' Sentence Transformers model
self.normalize_embeddings = True
self.received_loaded_model = False
# need to parameterize the embedding dims based on model config
if not embedding_dims:
self.embedding_dims = 768
if model_name == 'mini-lm-sbert':
self.embedding_dims = 384
else:
self.embedding_dims = embedding_dims
self.model_repo_location = LLMWareConfig.get_model_repo_path()
self.model_size="standard"
if model_name == 'mini-lm-sbert':
self.model_size = "mini"
self.transformer_base_model = None
self.sentence = None
if model:
logger.info("update: SemanticEmbedding model received model - will attempt to load as "
"Sentence Transformer model")
self.model = model
self.received_loaded_model = True
if len(model) >= 2:
try:
# general case is that embedding dimension is the "word_embedding_dimension" of the
# 'Pooling' layer, which is generally the second and last layer of the sbert model
self.embedding_dims = model[1].word_embedding_dimension
# there are at least 2 edge cases, in which a "Dense" layer is attached after the
# Pooling layer, and further consolidates the embeddings
if len(model) > 2:
logger.info("update: Sentence Transformer model with more than two layers - unusual - "
" depending upon the architecture, there may be issues loading the model- %s",
len(model))
# note: the most common case is with a Dense 3rd layer that maps the Pooling output to
# a different dimension - in this case - this should give the dimensions:
#
# last_layer_config = model[-1].get_config_dict()
# if "out_features" in last_layer_config:
# self.embedding_dims = last_layer_config["out_features"]
except:
logger.error("error: could not identify model to run embedding - ", model_name)
raise ModelNotFoundException(model_name)
if model_card and not model:
if "model_location" in model_card:
if model_card["model_location"] == "st_repo":
# try to pull the model and instantiate directly from Sentence Transformers
try:
from sentence_transformers import SentenceTransformer
except:
raise DependencyNotInstalledException("sentence_transformer")
try:
self.model = SentenceTransformer(model_card["model_name"])
except:
raise ModelNotFoundException(model_card["model_name"])
if "embedding_dims" in model_card:
self.embedding_dims = model_card["embedding_dims"]
else:
self.embedding_dims = self.model[1].word_embedding_dimension
def load_model_for_inference(self,fp=None, model_card=None, **kwargs):
""" This path has been deprecated starting with llmware 0.2.12. """
# if fp: self.model_repo_location = fp
raise LLMWareException(message="Exception - this load option has been deprecated. LLMWareSemanticModels "
"should be pulled from a sentence transformer standard repository.")
def embedding(self, sentence):
self.sentence = sentence
# preview before creating embedding
self.preview()
# embedding = self.model.encode(sentence, convert_to_tensor=True)
embedding = self.model.encode(self.sentence)
# add normalization for imported sentence transformer models
"""
if self.received_loaded_model and self.normalize_embeddings:
# normalize embeddings
embedding = torch.tensor(embedding).squeeze(0)
embedding = torch.nn.functional.normalize(embedding, p=2, dim=1)
embedding = embedding.detach().numpy()
"""
# embedding_2d = embedding.unsqueeze(0)
return embedding
def cosine_similarity(self, a, b):
return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
def euclidean_distance(self,a,b):
# aligning with FAISS - which returns square of Euclidean distance
return np.linalg.norm(a - b) * np.linalg.norm(a-b)
class ModelResources:
""" ModelResources is a global state mechanism used in conjunction with deploying the LLMWare Inference
Server class. It manages the persistent loading of multiple models behind the server. """
class _ModelState:
models_loaded = 0
models_list = []
@classmethod
def load_model(cls, model_name, sample=False, temperature=0.0, get_logits=True, max_output=200, api_key=None,
use_gpu=True):
model_card = ModelCatalog().lookup_model_card(model_name)
if model_card and model_name not in cls._ModelState.models_list:
setattr(cls._ModelState, model_name, ModelCatalog().load_model(model_name, api_key=api_key,
sample=sample, use_gpu=use_gpu,
get_logits=get_logits, max_output=max_output,
temperature=temperature))
cls._ModelState.models_list.append(model_name)
cls._ModelState.models_loaded += 1
logger.info(f"update: ModelResources - {cls._ModelState.models_loaded} - "
f"{cls._ModelState.models_list}")
@classmethod
def unload_model(cls, model_name):
""" Not implemented currently. """
return 0
@classmethod
def check_if_model_loaded(cls, model_name):
""" Utility method that checks if the model has already been loaded. """
if model_name in cls._ModelState.models_list:
return True
return False
@classmethod
def fetch_model(cls, model_name):
""" Returns the instantiated model that is already loaded in memory. """
return getattr(cls._ModelState, model_name)
class MultiModalModel:
"""A class to handle multi-modal models, supporting text, image, and other data types."""
def __init__(self, model_name, model_type, preprocessors=None, postprocessors=None):
self.model_name = model_name
self.model_type = model_type
self.preprocessors = preprocessors or {}
self.postprocessors = postprocessors or {}
def add_preprocessor(self, data_type, preprocessor):
"""Add a preprocessor for a specific data type."""
self.preprocessors[data_type] = preprocessor
def add_postprocessor(self, data_type, postprocessor):
"""Add a postprocessor for a specific data type."""
self.postprocessors[data_type] = postprocessor
def preprocess(self, data_type, data):
"""Preprocess data based on its type."""
if data_type in self.preprocessors:
return self.preprocessors[data_type](data)
return data
def postprocess(self, data_type, data):
"""Postprocess data based on its type."""
if data_type in self.postprocessors:
return self.postprocessors[data_type](data)
return data
def inference(self, inputs):
"""Perform inference on multi-modal inputs."""
processed_inputs = {
data_type: self.preprocess(data_type, data)
for data_type, data in inputs.items()
}
# Placeholder for model inference logic
raw_outputs = self._run_model(processed_inputs)
return {
data_type: self.postprocess(data_type, output)
for data_type, output in raw_outputs.items()
}
def _run_model(self, inputs):
"""Run the model on preprocessed inputs based on the model type."""
if not hasattr(self, 'model') or self.model is None:
raise ValueError("Model is not loaded. Please load a model before running inference.")
if self.model_type == "pytorch":
# PyTorch inference
import torch
input_tensors = {
data_type: torch.tensor(data) if isinstance(data, list) else torch.from_numpy(data)
for data_type, data in inputs.items()
}
with torch.no_grad():
outputs = {
data_type: self.model(input_tensor.unsqueeze(0))
for data_type, input_tensor in input_tensors.items()
}
return {data_type: output.squeeze(0).numpy() for data_type, output in outputs.items()}
elif self.model_type == "onnx":
# ONNX inference
import onnxruntime as ort
session = ort.InferenceSession(self.model)
outputs = {
data_type: session.run(None, {session.get_inputs()[0].name: data})[0]
for data_type, data in inputs.items()
}
return outputs
elif self.model_type == "openvino":
# OpenVino inference
from openvino.runtime import Core
core = Core()
compiled_model = core.compile_model(self.model, "CPU")
outputs = {
data_type: compiled_model([data])[0]
for data_type, data in inputs.items()
}
return outputs
elif self.model_type == "gguf":
# GGUF inference (example placeholder)
# Assuming GGUF uses a specific library for inference
from llmware.gguf_configs import GGUFInference
gguf_inference = GGUFInference(self.model)
outputs = {
data_type: gguf_inference.run(data)
for data_type, data in inputs.items()
}
return outputs
elif self.model_type == "tensorflow":
# TensorFlow inference
import tensorflow as tf
input_tensors = {
data_type: tf.convert_to_tensor(data) if isinstance(data, list) else tf.constant(data)
for data_type, data in inputs.items()
}
outputs = {
data_type: self.model(input_tensor[None, ...])
for data_type, input_tensor in input_tensors.items()
}
return {data_type: output.numpy() for data_type, output in outputs.items()}
else:
raise ValueError(f"Unsupported model type: {self.model_type}")
class PyTorchLoader:
""" PyTorchLoader is a wrapper class that consolidates all of the PyTorch model loading functions
throughout llmware - and provides the ability to create a single custom loader function to over-ride
the default PyTorch model loading, which relies upon HuggingFace repositories, and the formalisms
provided by the transformers library in terms of configs and model class code. This also enables a single
point to customize the behavior of transformers configurations. """
def __init__(self, api_key=None, trust_remote_code=True,custom_loader=None):
self.model_name = None
self.api_key=api_key
self.trust_remote_code = trust_remote_code
self.custom_loader = custom_loader
def get_generative_model(self, model_name, **kwargs):
""" Retrieves and instantiates a Pytorch Generative model. Takes a model_name as input, which is
assumed to map to the Huggingface repository name - this name is not necessarily the same as the
LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here
should be the hf_repo attribute on the model card. """
# will return None if no model found
model = None
self.model_name=model_name
if self.custom_loader:
model = self.custom_loader.loader(self.model_name,
self.api_key,self.trust_remote_code,caller="generative_model",**kwargs)
else:
try:
# will wrap in Exception if import fails
from transformers import AutoModelForCausalLM, AutoTokenizer
except ImportError:
raise DependencyNotInstalledException("transformers")
# insert dynamic pytorch load here
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
logger.debug("Pytorch loader - local dynamic load of torch here")
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
if self.api_key:
if torch.cuda.is_available():
model = AutoModelForCausalLM.from_pretrained(model_name, token=self.api_key,
trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModelForCausalLM.from_pretrained(model_name, token=self.api_key,
trust_remote_code=self.trust_remote_code)
else:
if torch.cuda.is_available():
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=self.trust_remote_code)
return model
def get_embedding_model(self, model_name, **kwargs):
""" Retrieves and instantiates a Pytorch Embedding model. Takes a model_name as input, which is
assumed to map to the Huggingface repository name - this name is not necessarily the same as the
LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here
should be the hf_repo attribute on the model card. """
model = None
self.model_name = model_name
if self.custom_loader:
model = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code, self.custom_loader,
caller="embedding_model", **kwargs)
else:
try:
# will wrap in Exception if import fails
from transformers import AutoModel
except ImportError:
raise DependencyNotInstalledException("transformers")
# insert dynamic pytorch load here
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
logger.debug("Pytorch loader - local dynamic load of torch here")
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
if self.api_key:
if torch.cuda.is_available():
model = AutoModel.from_pretrained(model_name, token=self.api_key,
trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModel.from_pretrained(model_name, token=self.api_key,
trust_remote_code=self.trust_remote_code)
else:
if torch.cuda.is_available():
model = AutoModel.from_pretrained(model_name, trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModel.from_pretrained(model_name, trust_remote_code=self.trust_remote_code)
return model
def get_reranker_model(self, model_name, **kwargs):
""" Retrieves and instantiates a Pytorch Reranker model. Takes a model_name as input, which is
assumed to map to the Huggingface repository name - this name is not necessarily the same as the
LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here
should be the hf_repo attribute on the model card. """
model = None
self.model_name = model_name
if self.custom_loader:
model = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code, self.custom_loader,
caller="reranker_model", **kwargs)
else:
try:
# will wrap in Exception if import fails
from transformers import AutoModelForSequenceClassification
except ImportError:
raise DependencyNotInstalledException("transformers")
# insert dynamic pytorch load here
global GLOBAL_TORCH_IMPORT
if not GLOBAL_TORCH_IMPORT:
logger.debug("Pytorch loader - local dynamic load of torch here")
if util.find_spec("torch"):
try:
global torch
torch = importlib.import_module("torch")
GLOBAL_TORCH_IMPORT = True
except:
raise LLMWareException(message="Exception: could not load torch module.")
else:
raise LLMWareException(message="Exception: need to import torch to use this class.")
if self.api_key:
if torch.cuda.is_available():
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=1,
token=self.api_key,
trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModelForSequenceClassification.from_pretrained(model_name,
num_labels=1,
token=self.api_key,
trust_remote_code=self.trust_remote_code)
else:
if torch.cuda.is_available():
model = AutoModelForSequenceClassification.from_pretrained(model_name,
num_labels=1,
trust_remote_code=self.trust_remote_code,
torch_dtype="auto")
else:
model = AutoModelForSequenceClassification.from_pretrained(model_name,
num_labels=1,
trust_remote_code=self.trust_remote_code)
return model
def get_tokenizer(self, model_name, **kwargs):
""" Retrieves and instantiates a tokenizer. Takes a model_name as input, which is
assumed to map to the Huggingface repository name - this name is not necessarily the same as the
LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here
should be the hf_repo attribute on the model card. """
tokenizer = None
self.model_name = model_name
if self.custom_loader:
tokenizer = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code,
self.custom_loader, caller="tokenizer", **kwargs)
else:
try:
# will wrap in Exception if import fails
from transformers import AutoTokenizer
except ImportError:
raise DependencyNotInstalledException("transformers")
if self.api_key:
tokenizer = AutoTokenizer.from_pretrained(model_name, token=self.api_key,
trust_remote_code=self.trust_remote_code)
else:
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=self.trust_remote_code)
return tokenizer
class CustomPTLoader:
""" CustomPTLoader is a stub class that demonstrates how to create a custom PT loader method that
can be passed to PyTorchLoader to over-ride the default load from a HuggingFace repository using transformers. """
def __init__(self, model_name=None, api_key=None, trust_remote_code=True,caller=None):
self.model_name = model_name
self.api_key= api_key
self.trust_remote_code = trust_remote_code
self.caller = caller
def loader(self, model_name,api_key=None, trust_remote_code=True, caller=None):
self.model_name = model_name
self.api_key= api_key
self.trust_remote_code=trust_remote_code
self.caller = caller
if self.caller == "generative_model":
return self.load_generative_model()
if self.caller == "embedding_model":
return self.load_embedding_model()
if self.caller == "tokenizer":
return self.load_tokenizer()
def load_generative_model(self):
""" Stub method to enable a custom loading of a generative PyTorch model. """
model=None
return model
def load_embedding_model(self):
""" Stub method to enable a custom loading of an embedding PyTorch model. """
model=None
return model
def load_tokenizer(self):
""" Stub method to enable a custom loading a tokenizer. """
tokenizer=None
return tokenizer
class WindowsLocalFoundryHandler:
""" Main handler for interface with Windows Local Foundry integration.
Model inferencing handled by implementation of WindowsLocalFoundryModel,
which subclasses BaseModel and mirrors closely the OpenAIModel class. """
def __init__(self):
self.model_id = ""
self.api_key = ""
self.base_url = None
def get_manager(self):
""" Checks if manager instance already created, and if not, creates new one.
This is the single entry point to get access to low level manager. """
foundry_mgr = _ModelRegistry().get_foundry_manager()
if not foundry_mgr:
try:
from foundry_local import FoundryLocalManager
except:
logger.warning(f"WindowsLocalFoundryHandler - could not "
f"load FoundryLocalManager SDK")
return None
# optional - check local uri
# from foundry_local.service import get_service_uri
# uri = get_service_uri()
# create new manager and save in ModelHQ state
foundry_mgr = _ModelRegistry().set_foundry_manager(FoundryLocalManager())
if foundry_mgr:
if hasattr(foundry_mgr, "endpoint"):
self.base_url = foundry_mgr.endpoint
return foundry_mgr
def activate_catalog(self, activate_status):
""" Connect with Windows Local Foundry, poll for latest model list
and activate in the LLMWare Model Catalog. """
result = True
mgr = self.get_manager()
if not mgr:
logger.info(f"Service not available - can not activate catalog")
activate_status = False
result = False
if activate_status:
if not self.is_server_started():
self.start_server()
# get available models + create ext catalog
model_list = self.create_model_catalog_extension()
for model in model_list:
_ModelRegistry().add_model(model)
mn = model.get("model_name", "")
logger.info(f"WindowsLocalFoundryManager - adding foundry model - {mn}")
else:
# remove instance from state
_ModelRegistry().reset_foundry_manager()
return result
def test_foundry(self):
""" Confirm that server has started and is running. """
mgr = self.get_manager()
if not mgr:
explanation = ("LocalFoundry Manager could not be created - "
"service does not appear to be available.")
return False, explanation
started = self.is_server_started()
if started:
return True, "Server has started"
else:
# not started
pass
if mgr:
if hasattr(mgr, "endpoint"):
self.base_url = mgr.endpoint
if hasattr(mgr, "api_key"):
self.api_key = mgr.api_key
return True, "Server Available but not Started"
def start_server_if_needed(self):
""" Start Windows Local Foundry server, if needed. """
if not self.is_server_started():
self.start_server()
return True
def is_server_started(self):
""" Check if Windows Local Foundry server has been started. """
mgr = self.get_manager()
started = False
if mgr:
started = mgr.is_service_running()
return started
def start_server(self):
""" Start Windows Local Foundry server. """
mgr = self.get_manager()
x = mgr.start_service()
return True
def stop_server(self):
""" Stop Windows Local Foundry server. """
import subprocess
cmd_args = "foundry service stop"
try:
subprocess.Popen(cmd_args, shell=True)
logger.info(f"WindowsLocalFoundryModel - server "
f"stopped successfully")
except:
logger.info(f"WindowsLocalFoundryModel - tried to stop server - "
f"unsuccessful - skipping")
return True
def check_if_cached(self, model_name):
""" Check if model is cached in .foundry locally """
is_cached = False
if model_name.endswith("-foundry"):
model_name = model_name[0:-len("-foundry")]
mgr = self.get_manager()
if not mgr:
return False
cached_models = mgr.list_cached_models()
# check if selected model in cache
for model in cached_models:
# model_id = model.id
# model_alias = model.alias
if model_name in [model.id, model.alias]:
is_cached = True
break
return is_cached
def download_if_needed(self, model_name):
""" Download local foundry manager, if not cached """
is_cached = self.check_if_cached(model_name)
if not is_cached:
confirmation = self.download_model(model_name)
return True
def download_model(self, model_name):
""" Download model through Windows Local Foundry """
# Download and load a model
mgr = self.get_manager()
model_info = mgr.download_model(model_name)
return model_info
def load_model(self, model_name,auto_unload=True):
""" Load model from local .foundry cache """
mgr = self.get_manager()
mgr.load_model(model_name)
return True
def unload_model(self, model_name):
""" Unload model from Windows Local Foundry """
mgr = self.get_manager()
mgr.unload_model(model_name)
return True
def list_all_models(self):
""" List all models available in Foundry Local repository """
# List available models in the catalog
mgr = self.get_manager()
catalog = mgr.list_catalog_models()
# alias | id | version | device_type | runtime | uri | file_size prompt_template | prompt
# device_type - CPU | GPU | NPU
return catalog
def release_all_models(self):
""" Release all models from Windows Local Foundry """
# safety check for Windows Local Foundry
response = False
if self.is_server_started():
response = True
mgr = self.get_manager()
list_loaded_models = mgr.list_loaded_models()
if list_loaded_models:
for model in list_loaded_models:
mgr.unload_model(model.id, force=True)
logger.info(f"release_all_models - unloading model - {model.id}")
list_loaded_models = mgr.list_loaded_models()
logger.info(f"release_all_models - updated loaded models - "
f"{list_loaded_models}")
return response
def _estimate_params(self, file_size_mb):
""" Quick estimation of the parameter count based on
binary file size of .foundry asset. """
# if no indicator found
default_params = 3
if file_size_mb >= 7000:
params = 14
return params
elif 3000 <= file_size_mb < 7000:
params = 7
return params
elif 1000 <= file_size_mb < 3000:
params = 3
return params
elif 100 <= file_size_mb < 1000:
params = 1
return params
else:
# default - unexpected
return default_params
def create_model_catalog_extension(self):
""" Create model catalog entry """
mc_ext = []
models = self.list_all_models()
names_only = []
for m in models:
new_card = {}
ctx = 8192
model_mb = m.file_size_mb
params = self._estimate_params(model_mb)
device_type = ""
if hasattr(m, "device_type"):
device_type = m.device_type
if device_type not in ["CPU", "GPU", "NPU"]:
if "npu" in m.id:
device_type = "NPU"
elif "gpu" in m.id:
device_type = "GPU"
elif "cpu" in m.id:
device_type = "CPU"
else:
device_type = "CPU"
if m.id not in names_only:
new_card.update({"model_name": m.id + "-foundry"})
# keep m.id for uniqueness (rather than m.alias)
new_card.update({"display_name": m.id + "-foundry"})
new_card.update({"model_family": "WindowsLocalFoundryModel"})
new_card.update({"model_category": "generative-api"})
new_card.update({"device_type": device_type})
new_card.update({"parameters": params})
new_card.update({"model_location": "api"})
new_card.update({"context_window": ctx})
new_card.update({"tags": ["llmware-chat", f"p{params}",
"windows_local_foundry",
"green", "emerald", "api"]})
mc_ext.append(new_card)
names_only.append(m.id)
return mc_ext
def list_all_cached_models(self):
""" List all locally cached models in .foundry """
model_list = []
# List models in cache
mgr = self.get_manager()
local_models = mgr.list_cached_models()
logger.info(f"Models in cache - {local_models}")
return model_list
class WindowsLocalFoundryModel(BaseModel):
""" WindowsLocalFoundryModel class implements the Windows Local Foundry API. """
def __init__(self, model_name=None, api_key=None, context_window=8192,
max_output=1000, temperature=0.0, **kwargs):
super().__init__(**kwargs)
logger.info(f"WindowsLocalFoundryModel - constructing model - {model_name}")
self.model_class = "WindowsLocalFoundryModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
# strip "-foundry" identifier
if model_name.endswith("-foundry"):
model_name = model_name[0:-len("-foundry")]
self.model_name = model_name
if api_key:
self.api_key = api_key
self.error_message = ("\nUnable to connect to WindowsLocalFoundry Model. "
"Please try again later.")
self.separator = "\n"
# assume input (50%) + output (50%)
self.max_total_len = context_window
self.max_input_len = int(context_window * 0.5)
self.llm_max_output_len = int(context_window * 0.5)
# inference settings
if temperature >= 0.0:
self.temperature = temperature
else:
self.temperature = 0.0
self.target_requested_output_tokens = max_output
self.add_prompt_engineering = False
self.add_context = ""
self.prompt = ""
self.context = ""
self.instruction_following = False
self.prompt_wrapper = None
# provides option to pass custom openai_client to
# model class at inference time
self.openai_client = None
if "model_card" in kwargs:
self.model_card = kwargs["model_card"]
else:
self.model_card = {}
self.available = True
self.manager = None
self.base_url = ""
self.api_key = ""
self.model_id = ""
self.prepare_foundry_manager_and_model()
self.post_init()
logger.info(f"WindowsLocalFoundryModel - constructed successfully")
def prepare_foundry_manager_and_model(self):
""" Consolidates all init steps around the foundry manager and model """
foundry_handler = WindowsLocalFoundryHandler()
mgr = foundry_handler.get_manager()
list_loaded_models = mgr.list_loaded_models()
loaded_model = None
if list_loaded_models:
for model in list_loaded_models:
mgr.unload_model(model.id, force=True)
logger.info(f"prepare_foundry_manager_and_model - unloading model - {model.id}")
list_loaded_models = mgr.list_loaded_models()
logger.info(f"prepare_foundry_manager_and_model - "
f"unloading model - {list_loaded_models}")
if loaded_model:
if loaded_model != self.model_name:
if mgr:
foundry_handler.unload_model(loaded_model)
if mgr:
self.available = True
foundry_handler.start_server_if_needed()
confirmation = foundry_handler.download_if_needed(self.model_name)
mgr.load_model(self.model_name)
self.manager = mgr
self.base_url = self.manager.endpoint
self.api_key = self.manager.api_key
self.model_id = self.manager.get_model_info(self.model_name).id
return True
def prompt_engineer_chatgpt3(self, query, context, inference_dict=None):
""" Builds prompt in ChatGPT format. """
if not self.add_prompt_engineering:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
else:
if context:
selected_prompt = "default_with_context"
else:
selected_prompt = "default_no_context"
prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt,
separator=self.separator,
query=query, context=context,
inference_dict=inference_dict)
system_message = prompt_dict["prompt_card"]["system_message"]
if not system_message:
system_message = "You are a helpful assistant."
system_instruction = None
if inference_dict:
if "system_instruction" in inference_dict:
system_instruction = inference_dict["system_instruction"]
if not system_instruction:
system_instruction = system_message
core_prompt = prompt_dict["core_prompt"]
messages = [
{"role": "system", "content": system_instruction},
{"role": "user", "content": core_prompt}
]
return messages
def prompt_engineer(self, query, context,inference_dict=None):
# unpack system instruction and chat history
messages = []
# this is the core message = context + query
if context:
output = context + "\n" + query
else:
output = query
chat_history = []
system_instruction = ""
if inference_dict:
if "chat_history" in inference_dict:
chat_history = inference_dict["chat_history"]
if "system_instruction" in inference_dict:
system_instruction = inference_dict["system_instruction"]
if not system_instruction:
system_instruction = "You are a helpful assistant."
# start with system message
messages.append({"role": "system", "content": system_instruction})
if chat_history:
for turn in chat_history:
messages.append({"role": "user", "content": turn["user"]})
messages.append({"role": "assistant",
"content": turn["assistant"]})
messages.append({"role": "user", "content": output})
return messages
def load_model_for_inference(self):
""" Check if model available, and if not load """
confirmation = WindowsLocalFoundryHandler().download_if_needed(self.model_name)
return True
def unload_model(self, model_name):
foundry_name = model_name
if model_name.endswith("-foundry"):
foundry_name = model_name[0:-len("-foundry")]
try:
from foundry_local import FoundryLocalManager
response = FoundryLocalManager().unload_model(foundry_name, force=True)
logger.info(f"WindowsLocalFoundryModel - "
f"successful unload model")
except:
logger.info(f"WindowsLocalFoundryModel - unload not successful - "
f"skipping")
return True
def close(self):
logger.info(f"WindowsLocalFoundryModel close model - {self.model_name}")
foundry_name = self.model_name
if self.model_name.endswith("-foundry"):
foundry_name = self.model_name[0:-len("-foundry")]
try:
response = self.manager.unload_model(foundry_name, force=True)
logger.info(f"WindowsLocalFoundryModel - "
f"successful unload model")
except:
logger.info(f"WindowsLocalFoundryModel - unload not successful - "
f"skipping")
return True
def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes inference on OpenAI Model. Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference. """
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if "openai_client" in inference_dict:
self.openai_client = inference_dict["openai_client"]
from llmware.configs import OpenAIConfig
# call to preview hook (not implemented by default)
self.preview()
# start change here
prompt_enriched = self.prompt_engineer(prompt,add_context,
inference_dict=inference_dict)
# new - change with openai v1 api
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
usage = {}
time_start = time.time()
# Configure the client to use the local Foundry service
client = OpenAI(base_url=self.base_url, api_key=self.api_key)
if self.model_name.endswith("-foundry"):
model_name = self.model_name[0:-(len("-foundry"))]
else:
model_name = self.model_name
# start here
# Set the model to use and generate a streaming response
stream = client.chat.completions.create(
model=self.model_id,
# messages=[{"role": "user", "content": prompt_enriched}],
messages=prompt_enriched,
stream=True,
max_tokens=self.target_requested_output_tokens
)
text_out = ""
prompt_tokens = 0
completion_tokens = 0
# Print the streaming response
for chunk in stream:
if chunk.choices[0].delta.content is not None:
token = chunk.choices[0].delta.content or ""
# print(chunk.choices[0].delta.content, end="", flush=True)
text_out += token
# yield token
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None,
api_key=None):
""" Executes stream inference on Windows Local Foundry Model with
OpenAI-compatible API.
Only required input is text-based prompt, with optional
parameters to "add_context" passage that will be assembled using the prompt style in the
"add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration,
and optional passing of api_key at time of inference.
"""
self.available = True
if not self.available:
logger.warning(f"WindowsLocalFoundryModel - could not connect to service - "
f"unfortunately, model is not available.")
usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens",
"processing_time": 0.0}
output_response = {"llm_response": "Service Not Available",
"usage": usage}
return output_response
self.prompt = prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
if "openai_client" in inference_dict:
self.openai_client = inference_dict["openai_client"]
from llmware.configs import OpenAIConfig
# call to preview hook (not implemented by default)
self.preview()
# default case - pass the prompt received without change
# prompt_enriched = self.prompt
prompt_enriched = self.prompt_engineer(prompt,add_context,
inference_dict=inference_dict)
logger.info(f"WindowsLocalFoundryModel - stream - created prompt - "
f"starting stream")
# new - change with openai v1 api
try:
from openai import OpenAI
except ImportError:
raise DependencyNotInstalledException("openai >= 1.0")
usage = {}
time_start = time.time()
# Configure the client to use the local Foundry service
client = OpenAI(base_url=self.base_url, api_key=self.api_key)
# Set the model to use and generate a streaming response
stream = client.chat.completions.create(
model=self.model_id,
# messages=[{"role": "user", "content": prompt_enriched}],
messages=prompt_enriched,
stream=True,
max_tokens=self.target_requested_output_tokens
)
text_out = ""
prompt_tokens = 0
completion_tokens = 0
# Print the streaming response
for chunk in stream:
if chunk.choices[0].delta.content is not None:
token = chunk.choices[0].delta.content or ""
# print(chunk.choices[0].delta.content, end="", flush=True)
text_out += token
yield token
usage = {"input": prompt_tokens,
"output": completion_tokens,
"total": prompt_tokens + completion_tokens,
"metric": "tokens",
"processing_time": time.time() - time_start}
output_response = {"llm_response": text_out, "usage": usage}
# output inference parameters
self.llm_response = text_out
self.usage = usage
self.logits = None
self.output_tokens = None
self.final_prompt = prompt_enriched
self.register()
return output_response
class ONNXEmbeddingModel(BaseModel):
""" ONNXEmbeddingModel class implements support for onnxruntime reranking,
and classifier models. Despite the name, true batch 'embedding' method is
not yet implemented but is on roadmap.
This is intended to be a simple interface to use encoder-based models in ONNX,
especially for on-device use. """
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None,
model_card=None, embedding_dims=None, max_len=None,
device="CPU", **kwargs):
super().__init__(**kwargs)
self.model_class = "ONNXEmbeddingModel"
self.model_category = "embedding"
self.model_name = model_name
self.model = model
self.tokenizer = tokenizer
self.embedding_dims = embedding_dims
self.model_type = None
self.max_total_len = 512
self.model_architecture = None
self.model_card = model_card
self.safe_buffer = 12
self.device = device
self.context_window = 512
# main handler for model inference session
self.ort_session = None
if self.model_card:
if "embedding_dims" in self.model_card:
self.embedding_dims = self.model_card["embedding_dims"]
if "context_window" in self.model_card:
self.context_window = self.model_card["context_window"]
self.use_gpu = False
self.api_key = api_key
if self.context_window > self.safe_buffer:
self.max_len = self.context_window - self.safe_buffer
else:
self.max_len = self.context_window
if max_len:
if max_len:
if max_len < self.context_window:
self.max_len = max_len
self.text_sample = None
self.model_folder_path = None
global GLOBAL_ONNX_CORE_RUNTIME
if not GLOBAL_ONNX_CORE_RUNTIME:
if util.find_spec("onnxruntime"):
# note: we import the pybind11 c++ wrapper interface directly
try:
global ort
ort = importlib.import_module("onnxruntime.capi.onnxruntime_pybind11_state")
GLOBAL_ONNX_CORE_RUNTIME = True
except:
raise LLMWareException(message="ONNXEmbeddingModel: could not load onnxruntime module. "
"If you have pip installed the library, then please check "
"that your platform is supported by onnxruntime.")
else:
raise LLMWareException(message="ONNXEmbeddingModel: need to import "
"onnxruntime to use this class, e.g., 'pip3 install "
"onnxruntime`")
# end dynamic import here
# self.post_init()
def load_model_for_inference(self, loading_directions, model_card=None):
""" Instantiates and loads model from local cache. """
if model_card:
self.model_card = model_card
# onnx expects a string path
self.model_folder_path = loading_directions
# instantiate the tokenizer from tokenizer.json file
# using hf tokenizers library
from tokenizers import Tokenizer
tokenizer_fn = "tokenizer.json"
self.tokenizer = Tokenizer.from_file(os.path.join(loading_directions, tokenizer_fn))
# currently hard-coded - adjust settings to increase size of text
self.tokenizer.enable_padding(length=150)
self.tokenizer.enable_truncation(150)
# currently hard-coded - load model.onnx file
model_fn = "model.onnx"
onnx_model_path = os.path.join(loading_directions, model_fn)
# create and initialize InferenceSession in onnxruntime
# -- calling methods directly in the pybind c++ .pyd file
session_options = ort.get_default_session_options()
self.ort_session = ort.InferenceSession(session_options, onnx_model_path, True, False)
# TODO: add more options and configs around providers and provider options
providers = []
provider_options = [dict()]
disabled_optimizers = set()
self.ort_session.initialize_session(providers, provider_options, disabled_optimizers)
# end - created and initialized onnxruntime session
return self
@staticmethod
def sigmoid(x):
""" Utility function to return sigmoid """
return 1.0 / (1.0 + np.exp(-x))
def rank(self, query, text_results, api_key=None, text_index="text",
top_n=20, relevance_threshold=None, min_return=3):
""" Executes reranking inference. """
# call to preview (not implemented by default)
# self.preview()
batches = []
if len(text_results) <= 32:
# need to package in chunks
batches.append(text_results)
else:
batch_count = len(text_results) // 32
if len(text_results) > batch_count * 32:
batch_count += 1
for x in range(0, batch_count):
stopper = min(len(text_results), (x + 1) * 32)
new_batch = text_results[x * 32:stopper]
batches.append(new_batch)
output = []
for batch in batches:
documents = []
for i, chunks in enumerate(batch):
documents.append(chunks[text_index])
# runs the inference to get similarity score
scores = self.compute_score(query, documents)
if not isinstance(scores, list):
scores = [scores]
for i, score in enumerate(scores):
batch[i].update({"rerank_score": score})
output.append(batch[i])
ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True)
# will return top_n if no relevance threshold set
if not relevance_threshold:
if top_n < len(ranked_output):
final_output = ranked_output[0:top_n]
else:
final_output = ranked_output
else:
final_output = []
# if relevance threshold, will return all results above threshold
for entries in ranked_output:
if entries["rerank_score"] >= relevance_threshold:
final_output.append(entries)
# fallback, if no result above threshold, then will return the min number of results
if len(final_output) == 0:
final_output = ranked_output[0:min_return]
self.register()
return final_output
def compute_score(self, query, documents, batch_size: int = 32):
""" Runs the core ranking inference to determine semantic similarity -
called by rank method """
sentence_pairs = [[query, doc] for doc in documents]
if isinstance(sentence_pairs[0], str):
sentence_pairs = [sentence_pairs]
self.tokenizer.enable_truncation(100)
self.tokenizer.enable_padding(pad_token="<pad>")
all_scores = []
for start_index in range(0, len(sentence_pairs), batch_size):
sentence_batch = sentence_pairs[start_index: start_index + batch_size]
input_ids = []
attn_mask = []
tokenizer_output = self.tokenizer.encode_batch(sentence_batch)
for sequence in tokenizer_output:
input_ids.append(sequence.ids)
attn_mask.append(sequence.attention_mask)
input_ids = np.array(input_ids, dtype=np.int64)
attn_mask = np.array(attn_mask, dtype=np.int64)
# onnxruntime - run inference session
output_names = [output.name for output in self.ort_session.outputs_meta]
# replace None with output_names
run_options = None
output = self.ort_session.run(output_names, {"input_ids": input_ids,
"attention_mask": attn_mask}, run_options)
# onnxruntime - end run inference session
scores = self.sigmoid(output[0])
if len(documents) == 1:
scores = [scores]
else:
score_float = []
# note: convert to 'float' -> safety for json conversion
for score in scores:
if isinstance(score, np.ndarray):
score_float.append(float(score[0]))
else:
score_float.append(float(score))
scores = score_float
all_scores.extend(scores)
return all_scores
def classify(self, text, **kwargs):
""" Executes a classifier inference with ONNX model """
config_path = os.path.join(self.model_folder_path, "config.json")
config = None
if os.path.exists:
try:
config = json.load(open(config_path, "r", errors="ignore"))
except:
logger.warning("onnx classifier config could not be loaded from file")
pass
if not config:
logger.warning("onnx classifier config - will not be able to convert outputs to keys - no config found.")
self.tokenizer.enable_truncation(300)
self.tokenizer.enable_padding(pad_token="<pad>")
tokenizer_output = self.tokenizer.encode(text)
input_ids = []
attn_mask = []
# for sequence in tokenizer_output:
input_ids.append(tokenizer_output.ids)
attn_mask.append(tokenizer_output.attention_mask)
input_ids = np.array(input_ids, dtype=np.int64)
attn_mask = np.array(attn_mask, dtype=np.int64)
# start here
output_names = [output.name for output in self.ort_session.outputs_meta]
# replace None with output_names
run_options = None
output = self.ort_session.run(output_names, {"input_ids": input_ids,
"attention_mask": attn_mask}, run_options)
scores = self.sigmoid(output[0])
dict_scores = {}
if config:
dict_scores = [{"label": config["id2label"][str(i)],
"score": score.item()} for i, score in enumerate(scores[0])]
dict_scores.sort(key=lambda x: x["score"], reverse=True)
self.register()
return dict_scores
class ONNXVisionGenerativeModel(BaseModel):
"""ONNXVisionGenerativeModel class implements the ONNX generative model API, with
integrated processor, to simplify multi-media processing. Currently this class
supports the phi-3-vision-onnx model by default, and images only.
Other multimedia types and additional model support - will be added over time. """
def __init__(self, model_name=None, api_key=None, model_card=None,
prompt_wrapper=None, instruction_following=False, context_window=2048,
use_gpu_if_available=True, trust_remote_code=True, sample=True, max_output=100, temperature=0.3,
get_logits=False, api_endpoint=None, **kwargs):
super().__init__()
self.model_class = "ONNXVisionGenerativeModel"
self.model_category = "generative"
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.final_prompt = None
self.model_name = model_name
self.hf_tokenizer_name = model_name
self.model = None
self.tokenizer = None
self.generator = None
self.sample = sample
self.get_logits = get_logits
self.auto_remediate_function_call_output = True
self.model_card = model_card
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.primary_keys = None
self.function = None
self.fc_supported = False
self.tool_type = None
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
# instantiate if model_name passed without actual model and tokenizer
if model_name and not api_endpoint:
if not self.model_card:
self.model_card = ModelCatalog().lookup_model_card(self.model_name)
if self.model_card:
if "hf_repo" in self.model_card:
hf_repo_name = self.model_card["hf_repo"]
self.hf_tokenizer_name = hf_repo_name
self.model = None
self.tokenizer = None
self.tokenizer_stream = None
# set to defaults for HF models in Model Catalog
# this can be over-ridden post initiation if needed for custom models
self.prompt_wrapper = "phi_3_vision"
self.instruction_following = False
# insert dynamic onnx load here
global GLOBAL_ONNX_GENAI_RUNTIME
if not GLOBAL_ONNX_GENAI_RUNTIME:
if util.find_spec("onnxruntime_genai"):
try:
global og
og = importlib.import_module("onnxruntime_genai")
GLOBAL_ONNX_GENAI_RUNTIME = True
except:
raise LLMWareException(message="ONNXVisionGenerativeModel: could not load onnxruntime_genai module. "
"If you have pip installed the library, then please check "
"that your platform is supported by onnxruntime.")
else:
import platform
if platform.system() == "Darwin":
raise LLMWareException(message=f"ONNXVisionGenerativeModel: identified current platform as 'Mac OS' "
f"which is not supported for onnxruntime_genai currently. "
f"\nWe would recommend using GGUF for generative inference on a "
f"Mac, or if you wish to use ONNXGenerativeModel, then please "
f"shift to a supported Windows or Linux platform.")
raise LLMWareException(message="ONNXVisionGenerativeModel: need to import "
"onnxruntime_genai to use this class, e.g., 'pip3 install "
"onnxruntime_genai`")
# end dynamic import here
self.params = None
self.prompt_wrapper = "phi_3_vision"
if not model_card:
# safety - empty iterable rather than 'None'
model_card = []
# deprecated attribute - will be removed in future releases
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
else:
self.instruction_following = False
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
self.trailing_space = ""
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
self.model_type = None
self.config = None
# parameters on context len + output generation
self.max_total_len = context_window
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
# key output parameters
self.max_output = max_output
self.target_requested_output_tokens = self.max_output
self.model_architecture = None
self.separator = "\n"
# use 0 as eos token id by default in generation -> but try to pull from model config
self.eos_token_id = 0
self.use_gpu = False
# no api key expected or required
self.api_key = api_key
self.error_message = "\nUnable to identify and load HuggingFace model."
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at default of 0.0
self.temperature = 0.0
self.add_prompt_engineering = False
self.add_context = ""
self.context = ""
self.prompt = ""
self.api_endpoint = api_endpoint
self.model_repo_path = None
self.model = None
self.processor = None
self.tokenizer_stream = None
# self.post_init()
def load_model_for_inference(self, loading_directions, model_card=None):
""" Loads ONNX Model from local path using loading directions. """
self.model_repo_path = loading_directions
if model_card:
self.model_card = model_card
self.model = og.Model(loading_directions)
logger.info("ONNXVisionGenerative Model - constructing model completed.")
try:
self.processor = self.model.create_multimodal_processor()
except Exception as e:
logger.warning(f"ONNXVisionGenerativeModel - failed to create multimodal "
f"processor with error code: {e}")
return self
self.tokenizer_stream = self.processor.create_stream()
return self
def unload_model(self):
""" Not implemented. """
return True
def set_api_key(self, api_key, env_var=""):
""" Not implemented for this model class """
return True
def _get_api_key(self, env_var=""):
""" Not implemented for this model class """
return True
def inference(self, text_prompt, image_path, **kwargs):
""" Vision inference expects two inputs -
-- text_prompt: instruction, e.g., 'describe this image'
-- image_path: full file path to supported image type (e.g., jpg, png)
"""
t0 = time.time()
if not self.processor:
logger.warning(f"ONNXVisionGenerativeModel - processor not created")
return ""
image_path = [image_path]
images = og.Images.open(*image_path)
# example prompt, e.g., phi-3-vision
# prompt = "<|user|>\n" + "<|image_1|>\n" + text_prompt + "<|end|>\n<|assistant|>\n"
prompt = PromptCatalog().apply_prompt_wrapper(text_prompt,self.prompt_wrapper)
try:
inputs = self.processor(prompt, images=images)
except Exception as e:
logger.info(f"ONNXVisionGenerativeModel - processor not successful - "
f"generated run time error - {e}")
inputs = []
logging.info("ONNXVisionGenerative Model - Generating response.")
params = og.GeneratorParams(self.model)
max_tokens = 7680
params.set_search_options(max_length=max_tokens)
generator = og.Generator(self.model, params)
generator.set_inputs(inputs)
token_count = 0
output_text = ""
while not generator.is_done():
generator.generate_next_token()
new_token = generator.get_next_tokens()[0]
new_token_dec = self.tokenizer_stream.decode(new_token)
output_text += new_token_dec
token_count += 1
if token_count > max_tokens:
break
logging.info(f"\nONNXVisionGenerativeModel - tokens generated: {token_count}")
logging.info(f"\nONNXVisionGenerative Model - processing time: {time.time()-t0}")
t1 = time.time()
# todo: will add separate counting of input tokens
input_token_count = 0
response = {"llm_response": output_text,
"usage": {"input": input_token_count,
"output": token_count,
"total": input_token_count +token_count,
"metric": "tokens",
"processing_time": t1-t0}}
return response
def stream(self, text_prompt, image_path, **kwargs):
""" Vision stream inference expects two inputs -
-- text_prompt: instruction, e.g., 'describe this image'
-- image_path: full file path to supported image type (e.g., jpg, png)
note: initial image encoding can easily take 10-20 seconds, depending upon system,
and then stream generation output is rapid after that.
"""
t0 = time.time()
if not self.processor:
logger.warning(f"ONNXVisionGenerativeModel - processor not created")
return ""
image_path = [image_path]
images = og.Images.open(*image_path)
# e.g., prompt for phi-3-vision currently
# prompt = "<|user|>\n" + "<|image_1|>\n" + text_prompt + "<|end|>\n<|assistant|>\n"
prompt = PromptCatalog().apply_prompt_wrapper(text_prompt, self.prompt_wrapper)
try:
inputs = self.processor(prompt, images=images)
except Exception as e:
logger.info(f"ONNXVisionGenerativeModel - processor not successful - "
f"generated run time error - {e}")
inputs = []
logging.info("ONNXVisionGenerative Model - Generating response.")
params = og.GeneratorParams(self.model)
max_tokens = 7680
params.set_search_options(max_length=max_tokens)
generator = og.Generator(self.model, params)
generator.set_inputs(inputs)
token_count = 0
output_text = ""
while not generator.is_done():
generator.generate_next_token()
new_token = generator.get_next_tokens()[0]
new_token_dec = self.tokenizer_stream.decode(new_token)
output_text += new_token_dec
token_count += 1
if token_count > max_tokens:
break
yield new_token_dec
logging.info(f"\nONNXVisionGenerativeModel - tokens generated: {token_count}")
logging.info(f"\nONNXVisionGenerative Model - processing time: {time.time()-t0}")
self.register()
return output_text
def cleanup_stream_gen_on_early_stop(self):
self.generator = None
return True
def register_top_logits(self, logit):
""" Gets the top logits and keeps a running log for output analysis. """
# logit will be in form of (1,1,vocab_len), for all but the first logit
# if first logit (will have shape of context len - add [-1])
if logit.shape[1] > 1:
# used for first logit with shape, e.g., (1,input_token_len,vocab_size)
logit_array = logit.squeeze()[-1]
else:
# all other logits after the first token
logit_array = logit.squeeze()
logit_size = logit.shape[-1]
# useful check on shape of logit_array
logit_array_size = logit_array.shape
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
for x in range(0, self.top_logit_count):
# round the float number to 3 digits
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3))
top_logits.append(pair)
self.logits_record.append(top_logits)
return top_logits
class _OVInfer:
""" Wrapper to package inputs and outputs in connection with executing a
forward pass on OpenVINO model (e.g., infer_request) - derived closely from
utilities provided in OpenVINO, e.g.:
https://github.com/openvinotoolkit/openvino/blob/master/src/bindings/python/src/openvino/utils/data_helpers/data_dispatcher.py
Not intended to be called directly, but is used as utility within other
OV model classes.
"""
def __init__(self, _infer_request=None):
self._infer_request = _infer_request
def ov_core_inference(self, inputs,
_infer_request,
share_outputs=False,
decode_strings=True):
""" Primary entrypoint into _OVInfer - takes the 'raw' inputs and
infer_request instance, and wraps both the inputs, calls the
forward pass on the infer request, and then wraps the outputs. """
self._infer_request = _infer_request
if inputs is None:
inputs = {}
# by default
is_shared = True
# prepare model inputs
if is_shared:
model_inputs = self._create_shared(inputs, _infer_request)
else:
model_inputs = self._create_copied(inputs, _infer_request)
# run inference
response = _infer_request.infer(model_inputs,
share_outputs=share_outputs,
decode_strings=decode_strings)
# package up response
ov_dict = OVDict(response)
return ov_dict
def _create_shared(self, inputs, request):
if isinstance(inputs, dict) or isinstance(inputs, tuple) or isinstance(inputs, OVDict):
inp_n = self.normalize_arrays(inputs, is_shared=True)
return {k: self.value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in inp_n.items()}
elif isinstance(inputs, list):
if len(request.input_tensors) == 1:
is_single_input = True
else:
is_single_input = False
inputs_x = self.normalize_arrays(
[inputs] if is_single_input and self.is_list_simple_type(inputs) else inputs, is_shared=True)
return {k: self.value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in inputs_x.items()}
elif isinstance(inputs, np.ndarray):
inp = self.normalize_arrays(inputs, is_shared=True)
return self.value_to_tensor(inp, request=request, is_shared=True)
elif isinstance(inputs, int) or isinstance(inputs, float) or isinstance(inputs, str) \
or isinstance(inputs, bytes) or isinstance(inputs, ovc.Tensor) or isinstance(inputs, np.number):
return self.value_to_tensor(inputs, request=request, is_shared=True)
# Check the special case of the array-interface
if hasattr(inputs, "__array__"):
request._inputs_data = self.normalize_arrays(inputs, is_shared=True)
return self.value_to_tensor(request._inputs_data, request=request, is_shared=True)
# raise error if incompatible type
raise LLMWareException(message=f"_OVInfer - _created_share - "
f"incompatible inputs of type: {type(inputs)}")
def _create_copied(self, inputs, request):
if isinstance(inputs, dict) or isinstance(inputs, tuple) or isinstance(OVDict):
return self.update_inputs(self.normalize_arrays(inputs, is_shared=False), request)
elif isinstance(inputs, list):
return self.update_inputs(
self.normalize_arrays([inputs] if request._is_single_input() and self.is_list_simple_type(inputs) else inputs,
is_shared=False), request)
elif isinstance(inputs, np.ndarray):
self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key=None)
return {}
elif isinstance(inputs, ovc.Tensor) or isinstance(inputs, np.number) or isinstance(inputs, int) or \
isinstance(inputs, float) or isinstance(inputs, str) or isinstance(inputs, bytes):
return self.value_to_tensor(inputs, request=request, is_shared=False)
# Check the special case of the array-interface
if hasattr(inputs, "__array__"):
self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key=None)
return {}
# raise error if incompatible type
raise LLMWareException(message=f"_OVInfer - _created_copied - "
f"incompatible inputs of type: {type(inputs)}")
def get_request_tensor(self, request, key=None):
""" Retrieves the input tensor from a request instance. """
if key is None:
return request.get_input_tensor()
elif isinstance(key, int):
return request.get_input_tensor(key)
elif isinstance(key, (str, ovc.ConstOutput)):
return request.get_tensor(key)
else:
raise LLMWareException(message=f"_OVInfer - get_request_tensor - "
f"key type {type(key)} is not "
f"supported for Tensor key: {key}")
def value_to_tensor(self, value, request=None, is_shared: bool = False, key=None) -> None:
""" Converts value to OV tensor """
if isinstance(value, ovc.Tensor):
return value
elif isinstance(value, np.ndarray):
tensor = self.get_request_tensor(request, key)
tensor_type = tensor.get_element_type()
tensor_dtype = tensor_type.to_dtype()
# String edge-case, always copy.
# Scalars are also handled by C++.
if tensor_type == ovc.Type.string:
return ovc.Tensor(value, shared_memory=False)
# Scalars edge-case:
if value.ndim == 0:
tensor_shape = tuple(tensor.shape)
if tensor_dtype == value.dtype and tensor_shape == value.shape:
return ovc.Tensor(value, shared_memory=is_shared)
elif tensor.size == 0:
# the first infer request for dynamic input cannot reshape to 0 shape
return ovc.Tensor(value.astype(tensor_dtype).reshape((1)), shared_memory=False)
else:
return ovc.Tensor(value.astype(tensor_dtype).reshape(tensor_shape), shared_memory=False)
# WA for FP16-->BF16 edge-case, always copy.
if tensor_type == ovc.Type.bf16:
tensor = ovc.Tensor(tensor_type, value.shape)
tensor.data[:] = value.view(tensor_dtype)
return tensor
# WA for "not writeable" edge-case, always copy.
if value.flags["WRITEABLE"] is False:
tensor = ovc.Tensor(tensor_type, value.shape)
tensor.data[:] = value.astype(tensor_dtype) if tensor_dtype != value.dtype else value
return tensor
# If types are mismatched, convert and always copy.
if tensor_dtype != value.dtype:
return ovc.Tensor(value.astype(tensor_dtype), shared_memory=False)
# Otherwise, use mode defined in the call.
return ovc.Tensor(value, shared_memory=is_shared)
elif isinstance(value, list):
return ovc.Tensor(value)
elif isinstance(value, int) or isinstance(value, float) or isinstance(value, str) or \
isinstance(value, bytes) or isinstance(value, np.number):
# np.number/int/float/str/bytes edge-case, copy will occur in both scenarios.
tensor_type = self.get_request_tensor(request, key).get_element_type()
tensor_dtype = tensor_type.to_dtype()
tmp = np.array(value)
# String edge-case -- it converts the data inside of Tensor class.
# If types are mismatched, convert.
if tensor_type != ovc.Type.string and tensor_dtype != tmp.dtype:
return ovc.Tensor(tmp.astype(tensor_dtype), shared_memory=False)
return ovc.Tensor(tmp, shared_memory=False)
# raise error if incompatible type
raise LLMWareException(message=f"_OVInfer - value_to_tensor - "
f"incompatible inputs of type: {type(value)}")
def to_c_style(self, value: Any, is_shared: bool = False) -> Any:
if not isinstance(value, np.ndarray):
if hasattr(value, "__array__"):
return self.to_c_style(np.array(value, copy=False), is_shared) if is_shared else np.array(value, copy=True)
return value
return value if value.flags["C_CONTIGUOUS"] else np.ascontiguousarray(value)
def normalize_arrays(self, inputs: Any, is_shared: bool = False) -> Any:
if isinstance(inputs, dict):
return {k: self.to_c_style(v, is_shared) if is_shared else v for k, v in inputs.items()}
if isinstance(inputs, OVDict):
return {i: self.to_c_style(v, is_shared) if is_shared else v for i, (_, v) in enumerate(inputs.items())}
if isinstance(inputs, list) or isinstance(inputs, tuple):
return {i: self.to_c_style(v, is_shared) if is_shared else v for i, v in enumerate(inputs)}
if isinstance(inputs, np.ndarray):
return self.to_c_style(inputs, is_shared) if is_shared else inputs
# Check the special case of the array-interface
if hasattr(inputs, "__array__"):
return self.to_c_style(np.array(inputs, copy=False), is_shared) if is_shared else np.array(inputs, copy=True)
# raise error if incompatible type
raise LLMWareException(message=f"_OVInfer - normalize_arrays - "
f"incompatible inputs of type: {type(inputs)}")
def set_request_tensor(self, request, tensor, key=None) -> None:
if key is None:
request.set_input_tensor(tensor)
elif isinstance(key, int):
request.set_input_tensor(key, tensor)
elif isinstance(key, (str, ovc.ConstOutput)):
request.set_tensor(key, tensor)
else:
# raise error if incompatible type
raise LLMWareException(message=f"_OVInfer - set_request_tensor - "
f"unsupported key type: {type(key)} for "
f"tensor under key: {key}")
def update_tensor(self, inputs: Any, request, key=None) -> None:
if isinstance(inputs, np.ndarray):
if inputs.ndim != 0:
tensor = self.get_request_tensor(request, key)
# Update shape if there is a mismatch
if tuple(tensor.shape) != inputs.shape:
tensor.shape = inputs.shape
# When copying, type should be up/down-casted automatically.
if tensor.element_type == ovc.Type.string:
tensor.bytes_data = inputs
else:
tensor.data[:] = inputs[:]
else:
# If shape is "empty", assume this is a scalar value
self.set_request_tensor(
request,
self.value_to_tensor(inputs, request=request, is_shared=False, key=key),
key,
)
# TODO: what to return
elif isinstance(inputs, np.number) or isinstance(inputs, float) or isinstance(inputs, int) or \
isinstance(inputs, str):
self.set_request_tensor(
request,
self.value_to_tensor(inputs, request=request, is_shared=False, key=key),
key,
)
if hasattr(inputs, "__array__"):
self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key)
return None
# raise error if unsupported key type
raise LLMWareException(message=f"_OVInfer - update_tensor - "
f"unsupported key type: {type(inputs)} for "
f"tensor under key: {key}")
def update_inputs(self, inputs: dict, request):
# Create new temporary dictionary.
# new_inputs will be used to transfer data to inference calls,
# ensuring that original inputs are not overwritten with Tensors.
new_inputs = {}
for key, value in inputs.items():
if not isinstance(key, (str, int, ovc.ConstOutput)):
raise TypeError(f"Incompatible key type for input: {key}")
# Copy numpy arrays to already allocated Tensors.
# If value object has __array__ attribute, load it to Tensor using np.array
if isinstance(value, (np.ndarray, np.number, int, float, str)) or hasattr(value, "__array__"):
self.update_tensor(value, request, key)
elif isinstance(value, list):
new_inputs[key] = ovc.Tensor(value)
# If value is of Tensor type, put it into temporary dictionary.
elif isinstance(value, ovc.Tensor):
new_inputs[key] = value
# Throw error otherwise.
else:
# raise error if unsupported type
raise LLMWareException(message=f"_OVInfer - update_inputs - "
f"unsupported key type: {type(value)} for "
f"tensor under key: {key}")
return new_inputs
def is_list_simple_type(self, input_list: list) -> bool:
for sublist in input_list:
if isinstance(sublist, list):
for element in sublist:
if not isinstance(element, (str, float, int, bytes)):
return False
else:
if not isinstance(sublist, (str, float, int, bytes)):
return False
return True
class OVDict(Mapping):
""" Output handler for OV infer request forward pass, used for
downstream processing in OVEmbeddingModel class - mirrors
OpenVINO OVDict definition. """
def __init__(self, _dict):
self._dict = _dict
self._names = None
def __iter__(self):
return self._dict.__iter__()
def __len__(self) -> int:
return len(self._dict)
def __repr__(self) -> str:
return self._dict.__repr__()
def __get_names(self):
return {key: key.get_names() for key in self._dict.keys()}
def __get_key(self, index: int):
return list(self._dict.keys())[index]
def __getitem__(self, key) -> np.ndarray:
if isinstance(key, str):
if self._names is None:
self._names = self.__get_names()
for port, port_names in self._names.items():
if key in port_names:
return self._dict[port]
raise KeyError(key)
elif isinstance(key, int):
try:
return self._dict[self.__get_key(key)]
except IndexError:
raise KeyError(key)
else:
try:
return self._dict[key]
except:
raise LLMWareException(message=f"OVDict - unknown key type - {type(key)}")
def keys(self):
return self._dict.keys()
def values(self):
return self._dict.values()
def items(self):
return self._dict.items()
def names(self):
if self._names is None:
self._names = self.__get_names()
return tuple(self._names.values())
def to_dict(self):
return self._dict
def to_tuple(self):
return tuple(self._dict.values())
class OVEmbeddingModel(BaseModel):
""" OVEmbeddingModel class implements a high-level interface to use
OpenVINO encoder-based models, supporting three different modalities currently:
-- Embedding - for use with vector databases
-- Reranker - for in-memory semantic similarity comparisons
-- Classify - for classifier based models
"""
def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None,
embedding_dims=None, use_gpu_if_available=True, max_len=None, device="CPU", **kwargs):
super().__init__(**kwargs)
self.model_class = "OVEmbeddingModel"
self.model_category = "embedding"
self.model_name = model_name
self.model = model
self.tokenizer= tokenizer
self.embedding_dims = embedding_dims
self.model_type = None
self.max_total_len = 512
self.model_architecture = None
self.model_card = model_card
self.safe_buffer = 12
self.device = device
# default for HF embedding model -> will be over-ridden by model card / configs, if available
self.context_window = 512
if self.model_card:
if "embedding_dims" in self.model_card:
self.embedding_dims = self.model_card["embedding_dims"]
if "context_window" in self.model_card:
self.context_window = self.model_card["context_window"]
if "model_name" in self.model_card:
self.model_name = self.model_card["model_name"]
global ovc
global GLOBAL_OPENVINO_IMPORT
if not GLOBAL_OPENVINO_IMPORT:
if not util.find_spec("openvino"):
raise LLMWareException(message="OVEmbeddingModel: to use OVEmbeddingModel requires "
"install of 'openvino' library. "
"Please try: `pip3 install openvino` "
"and confirm that your "
"hardware platform is supported.")
if util.find_spec("openvino"):
# loads/accesses the openvino pybind pyd methods directly
try:
ovc = importlib.import_module("openvino._pyopenvino")
GLOBAL_OPENVINO_IMPORT = True
except:
raise LLMWareException(message="OVEmbeddingModel: could not load openvino module.")
if not ovc:
raise LLMWareException(message="OVEmbeddingModel: could not load required openvino dependency.")
# end dynamic import here
self.use_gpu = False
# no api key expected or required
self.api_key = api_key
# set max len for tokenizer truncation with 'safe_buffer' below context_window size
if self.context_window > self.safe_buffer:
self.max_len = self.context_window - self.safe_buffer
else:
self.max_len = self.context_window
# option to set smaller size than model context window
if max_len:
if max_len < self.context_window:
self.max_len = max_len
self.text_sample = None
self.model_folder_path = None
self._device = self.device
self.is_dynamic = True
self.read_model_xml_path = None
self.model = None
self.request = None
self._infer_request = None
self.input_names = None
self.output_names = None
self.config = None
# post init not implemented for this model class currently
# self.post_init()
def load_model_for_inference(self, loading_directions, model_card=None):
""" Loads OV Embedding Model from local path using loading directions. """
if model_card:
self.model_card = model_card
self.model_folder_path = Path(loading_directions)
# load the tokenizer from tokenizer.json in model repo
from tokenizers import Tokenizer
tokenizer_fn = "tokenizer.json"
self.tokenizer = Tokenizer.from_file(os.path.join(loading_directions, tokenizer_fn))
# hard-coded at 150 tokens -> adjust to increase/decrease
self.tokenizer.enable_padding(length=150)
self.tokenizer.enable_truncation(150)
if not ovc:
logger.warning("OVEmbeddingModel - could not find backend module")
return False
# need to get config.json file
self.config = self.get_config_from_file()
self.read_model_xml_path = Path(os.path.join(loading_directions, "openvino_model.xml"))
core = ovc.Core()
self.model = core.read_model(self.read_model_xml_path.resolve(),
self.read_model_xml_path.with_suffix(".bin").resolve())
if self.is_dynamic:
height = None
width = None
self.model = self._reshape(self.model, -1, -1, height, width)
input_names = {}
for idx, key in enumerate(self.model.inputs):
names = tuple(key.get_names())
input_names[next((name for name in names if "/" not in name), names[0])] = idx
self.input_names = input_names
output_names = {}
for idx, key in enumerate(self.model.outputs):
names = tuple(key.get_names())
output_names[next((name for name in names if "/" not in name), names[0])] = idx
self.output_names = output_names
self.request = None
if self.request is None:
# try to load on GPU first, and fallback to CPU, if GPU fails
try:
gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME")
logger.info(f"OVGenerativeModel - found gpu device - name: {gpu_device_name}.")
device = "GPU"
logger.info(f"OVEmbeddingModel - successful finding GPU")
except:
logger.debug("OVGenerativeModel - loading - could not find gpu - setting device for CPU")
device = "CPU"
self._device = device
logger.info(f"OVEmbeddingModel - device - {device}")
ov_config = {}
self.request = core.compile_model(self.model, self._device, ov_config)
logger.info(f"OVEmbedding - completed model compile - {self.model_name} "
f"on device - {self._device}")
return self
def get_config_from_file(self):
""" Loads config information from config.json file """
config_file = os.path.join(self.model_folder_path, "config.json")
try:
config = json.load(open(config_file, "r"))
except:
config = {}
return config
def _inference(self, inputs):
""" Internal inference method implements forward pass on the model """
if not self._infer_request:
self._infer_request = self.request.create_infer_request()
try:
outputs = _OVInfer().ov_core_inference(inputs,
self._infer_request,
share_outputs=False,
decode_strings=True)
except Exception as e:
raise LLMWareException(message=f"OVEmbeddingModel - _inference - "
f"unsuccessful - generated error code - "
f"{e}")
return outputs
def set_api_key(self, api_key, env_var=""):
""" Not implemented """
return True
def _get_api_key(self, env_var=""):
""" Not implemented """
return True
def token_counter(self, text_sample):
""" Counts tokens in text sample. """
toks = self.tokenizer.encode(text_sample).ids
return len(toks)
@staticmethod
def sigmoid(x):
"""Simple sigmoid function. Not numerically stable!"""
return 1.0 / (1.0 + np.exp(-x))
def _reshape(self, model, batch_size, sequence_length, height=None, width=None):
""" Internal implementation method to reshape the input """
shapes = {}
for inputs in model.inputs:
shapes[inputs] = inputs.get_partial_shape()
shapes[inputs][0] = batch_size
shapes[inputs][1] = sequence_length
if height is not None:
shapes[inputs][2] = height
if width is not None:
shapes[inputs][3] = width
model.reshape(shapes)
return model
def reshape(self, batch_size, sequence_length, height=None, width= None):
""" Reshape input """
self.is_dynamic = True if batch_size == -1 and sequence_length == -1 else False
self.model = self._reshape(self.model, batch_size, sequence_length, height, width)
self.request = None
return self
def forward(self, input_ids, attention_mask, token_type_ids = None, **kwargs):
""" Forward pass on model """
np_inputs = isinstance(input_ids, np.ndarray)
if not np_inputs:
input_ids = np.array(input_ids)
attention_mask = np.array(attention_mask)
token_type_ids = np.array(token_type_ids) if token_type_ids is not None else token_type_ids
inputs = {
"input_ids": input_ids,
"attention_mask": attention_mask,
}
# Add the token_type_ids when needed
if "token_type_ids" in self.input_names:
inputs["token_type_ids"] = token_type_ids if token_type_ids is not None else np.zeros_like(input_ids)
outputs = self._inference(inputs)
last_hidden_state = outputs["last_hidden_state"]
embedding = last_hidden_state[:,0]
return embedding
def classify(self, text,**kwargs):
""" Implements a classify inference for classifier-based models that
have been fine-tuned with a classifier head"""
self.text_sample = text
if not isinstance(self.text_sample, list):
self.text_sample = [self.text_sample]
input_ids = []
attn_mask = []
tokenizer_output = self.tokenizer.encode_batch(self.text_sample)
for sequence in tokenizer_output:
input_ids.append(sequence.ids)
attn_mask.append(sequence.attention_mask)
input_ids = np.array(input_ids)
attn_mask = np.array(attn_mask)
np_inputs = isinstance(input_ids, np.ndarray)
if not np_inputs:
input_ids = np.array(input_ids)
attn_mask = np.array(attn_mask)
inputs = {
"input_ids": input_ids,
"attention_mask": attn_mask,
}
# Add the token_type_ids when needed
if "token_type_ids" in self.input_names:
# may require customization for some model types
inputs["token_type_ids"] = np.zeros_like(input_ids)
outputs = self._inference(inputs)
logits = outputs["logits"]
max_value = np.max(logits, axis=-1, keepdims=True)
shifted_exp = np.exp(logits - max_value)
scores = shifted_exp / shifted_exp.sum(axis=-1, keepdims=True)
if "id2label" in self.config:
try:
dict_scores = [{"label": self.config["id2label"][str(i)],
"score": score.item()} for i, score in enumerate(scores[0])]
except:
dict_scores = [{"label": "NA", "score": 0.0}]
logger.info(f"OVEmbeddingModel - classify configs not resolved - {self.config} - "
f"{scores[0]}")
else:
# report scores without label if not available (e.g, missing config)
dict_scores = []
for i, score in enumerate(scores[0]):
new_entry = {"label": f"score_{i+1}", "score": score.item()}
dict_scores.append(new_entry)
dict_scores.sort(key=lambda x: x["score"], reverse=True)
self.register()
return dict_scores
def embedding (self, text_sample, api_key=None):
""" Executes embedding inference. """
self.text_sample = text_sample
# call to preview (not implemented by default)
# self.preview()
# return embeddings only
if not isinstance(self.text_sample,list):
self.text_sample = [self.text_sample]
input_ids = []
attn_mask = []
tokenizer_output = self.tokenizer.encode_batch(self.text_sample)
for sequence in tokenizer_output:
input_ids.append(sequence.ids)
attn_mask.append(sequence.attention_mask)
input_ids = np.array(input_ids)
attn_mask = np.array(attn_mask)
model_input = {"input_ids": input_ids, "attention_mask": attn_mask}
# Add the token_type_ids when needed
if "token_type_ids" in self.input_names:
model_input["token_type_ids"] = np.zeros_like(input_ids)
outputs = self._inference(model_input)
last_hidden_state = outputs["last_hidden_state"]
embedding = last_hidden_state[:,0]
# l2 normalization with numpy
embeddings_normalized = embedding / np.linalg.norm(embedding,2,axis=1,keepdims=True)
self.register()
return embeddings_normalized
def rank (self, query, text_results, text_index="text",
api_key=None, top_n=20, relevance_threshold=None, min_return=3):
""" Executes reranking inference. """
# call to preview (not implemented by default)
# self.preview()
batches = []
if len(text_results) <= 32:
# need to package in chunks
batches.append(text_results)
else:
batch_count = len(text_results) // 32
if len(text_results) > batch_count * 32:
batch_count += 1
for x in range(0,batch_count):
stopper = min(len(text_results), (x+1)*32)
new_batch = text_results[x*32:stopper]
batches.append(new_batch)
output = []
for batch in batches:
documents = []
for i, chunks in enumerate(batch):
documents.append(chunks[text_index])
scores = self.compute_score(query, documents)
if not isinstance(scores,list):
scores = [scores]
for i, score in enumerate(scores):
batch[i].update({"rerank_score": score})
output.append(batch[i])
ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True)
# will return top_n if no relevance threshold set
if not relevance_threshold:
if top_n < len(ranked_output):
final_output = ranked_output[0:top_n]
else:
final_output = ranked_output
else:
final_output = []
# if relevance threshold, will return all results above threshold
for entries in ranked_output:
if entries["rerank_score"] >= relevance_threshold:
final_output.append(entries)
# fallback, if no result above threshold, then will return the min number of results
if len(final_output) == 0:
final_output = ranked_output[0:min_return]
self.register()
return final_output
def compute_score(self, query, documents, batch_size: int = 32):
""" Applies semantic similarity ranker to query and a set of text chunks. """
sentence_pairs = [[query, doc] for doc in documents]
# if empty query, then return [] for empty scores
if len(sentence_pairs) == 0:
return []
assert isinstance(sentence_pairs, list)
if isinstance(sentence_pairs[0], str):
sentence_pairs = [sentence_pairs]
#TODO: look at truncation settings
self.tokenizer.enable_truncation(100)
self.tokenizer.enable_padding(pad_token="<pad>")
all_scores = []
for start_index in range(0, len(sentence_pairs), batch_size):
sentences_batch = sentence_pairs[start_index: start_index + batch_size]
input_ids = []
attn_mask = []
tokenizer_output = self.tokenizer.encode_batch(sentences_batch)
for sequence in tokenizer_output:
input_ids.append(sequence.ids)
attn_mask.append(sequence.attention_mask)
input_ids = np.array(input_ids)
attn_mask = np.array(attn_mask)
# note: last element is the 'position_type_ids'
model_input = (input_ids, attn_mask, np.zeros_like(input_ids))
scores = self._inference(model_input)
scores = self.sigmoid(scores["logits"].squeeze())
# safety check if single value, e.g., if input is only one document
if len(documents) == 1:
scores = [scores]
else:
scores = scores.tolist()
all_scores.extend(scores)
if len(all_scores) == 1:
all_scores = all_scores[0]
return all_scores
class GGUFVisionGenerativeModel(BaseModel):
""" Implementation of GGUF Vision Model class - instantiate and run vision-to-text inferences using
GGUF llama.cpp models with MTMD CLIP-based visual encoding - wraps two underlying models (which interact
directly with each other) -
-- decoder generative model, e.g., llama main
-- encoder clip model, e.g., mtmd clip model
"""
def __init__(self, model_name=None, model_card=None, api_key=None, prompt_wrapper=None, instruction_following=False,
context_window=2048, use_gpu_if_available=True, get_logits=False,
sample=True, max_output=500, temperature=0.0, api_endpoint=None, **kwargs):
super().__init__(**kwargs)
logger.debug("GGUFVisionGenerativeModel - initializing GGUF Vision model ... ")
self.model_class = "GGUFVisionGenerativeModel"
self.model_category = "generative"
# key model state attributes
self.gguf_file = None
self.gguf_repo = None
self.clip_file = None
self.clip_model = None
# main llama model
self._lib = None
self._model = None
self._ctx = None
self._batch = None
self.model_path = None
self.model_params = None
self.context_params = None
# attributes of mtmd backend lib and clip model
self._libmtmd = None
self.mtmd_ctx = None
self._clip_model = None
self.clip_ctx = None
self.clip_model_path = ""
self.clip_base_name = "mtmd"
self._clip_base_path = ""
self.llm_response = None
self.usage = None
self.logits = None
self.output_tokens = None
self.prompt = None
self.final_prompt = None
self._logits_all = False
# set verbose level in environ level - will be picked up by callback in llama_cpp & mtmd
# set to "ON" to view details for debugging
os.environ["llama_cpp_verbose"] = GGUFConfigs().get_config("llama_cpp_verbose")
# e.g., os.environ["llama_cpp_verbose"] = "ON"
self.use_sampling = sample
self.get_logits = get_logits
self.logits_record = []
self.output_tokens = []
self.top_logit_count = 10
self.auto_remediate_function_call_output = True
# default safety check in GGUF Configs that can be adjusted
gguf_configs_max = GGUFConfigs().get_config("max_output_tokens")
if max_output > gguf_configs_max:
# truncate max output to GGUFConfigs max
logger.warning(
f"GGUFVisionGenerativeModel - requested output len - {max_output} > {gguf_configs_max}, which is the "
f"current GGUF default max.\n--Truncating to {gguf_configs_max} output tokens.\n--Note: "
f"to change GGUF default max to new integer amount, say 500:\n "
f" GGUFConfigs().set_config(\"max_output_tokens\", 500)"
)
max_output = gguf_configs_max
self.max_output = max_output
self.n_seq_max = max_output
self.target_requested_output_tokens = self.n_seq_max
self.max_total_len = 2048
self.max_input_len = int(0.5 * context_window)
self.llm_max_output_len = int(0.5 * context_window)
self.max_output_len = self.n_seq_max
self.model_name = model_name
self.prompt_wrapper = prompt_wrapper
self.instruction_following = instruction_following
self.trailing_space = ""
self.separator = "\n"
self.eos_token_id = 0
self.add_prompt_engineering = False
self.add_context = ""
self.model_type = "gguf"
self.model_card = model_card
self.primary_keys = None
self.function = None
self.hf_tokenizer_name = None
self.fc_supported = False
if model_card:
if "primary_keys" in model_card:
self.primary_keys = model_card["primary_keys"]
if "function" in model_card:
self.function = model_card["function"]
if "tokenizer" in model_card:
self.hf_tokenizer_name = model_card["tokenizer"]
if "function_call" in model_card:
self.fc_supported = model_card["function_call"]
if "trailing_space" in model_card:
self.trailing_space = model_card["trailing_space"]
else:
self.trailing_space = ""
if "eos_token_id" in model_card:
self.eos_token_id = model_card["eos_token_id"]
if "context_window" in model_card:
self.max_total_len = model_card["context_window"]
if "prompt_wrapper" in model_card:
self.prompt_wrapper = model_card["prompt_wrapper"]
else:
self.prompt_wrapper = "human_bot"
if "gguf_file" in model_card:
self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf"
if "clip_file" in model_card:
self.clip_file = model_card["clip_file"]
if "gguf_repo" in model_card:
self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
if "instruction_following" in model_card:
self.instruction_following = model_card["instruction_following"]
# temperature configuration
# if temperature set at time of loading the model, then use that setting
if temperature != -99:
self.temperature = temperature
elif "temperature" in model_card:
# if not set, then pull the default temperature from the model card
self.temperature = model_card["temperature"]
else:
# if no guidance from model loading or model card, then set at GGUFConfigs default
self.temperature = GGUFConfigs().get_config("temperature_default")
# new option to 'force' use of cuda lib, and over-ride safety checks
if GGUFConfigs().get_config("force_gpu"):
self.use_gpu = True
else:
if sys.platform.lower() not in GGUFConfigs().get_config("cuda_platforms"):
self.use_gpu = False
else:
# min drivers set to the lowest level for CUDA 12.1 on Linux
min_drivers = [525, 60]
if sys.platform.lower() == "win32":
min_drivers = GGUFConfigs().get_config("cuda_windows_driver_min")
gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers)
# use_gpu set to TRUE only if:
# (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS
# (2) use_gpu set to True in GGUFConfigs
# (3) use_gpu_if_available flag set to True (by default)
# (4) cuda found and drivers current via direct polling of nvidia-smi executable in
# ModelCatalog.gpu_available method
self.use_gpu = (GGUFConfigs().get_config("use_gpu")
and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms")
and gpu_available["drivers_current"] and gpu_available["gpu_found"]
and use_gpu_if_available)
# set default minimum
self.n_batch = 2048 # alt/previous: 512
self.last_n_tokens_size = 64
self._n_vocab = None
self._n_ctx = None
self._token_nl = None
self._token_eos = None
self._candidates = None
self.input_ids = None
self.scores = None
self.n_tokens = 0
self.prev = []
self.grammar = None
for key, value in GGUFConfigs().get_sampling_params().items():
setattr(self, key, value)
# no api key expected or required
self.api_key = api_key
self.api_endpoint = api_endpoint
self.error_message = "\nUnable to identify and load GGUF Vision Generative model."
self.prompt = ""
self.context = ""
self.tool_type = None
self.model_repo_path = None
self._sampler = None
self._last_image_embed = None
self._last_image_hash = None
self.file_path = ""
self.vocab = None
# not implemented currently keeps list of tuples - (file_path, embed)
# roadmap - capture image embeddings separately for re-use
self.embed_list = []
self.embed_tokens = []
self.verbose = True
self.post_init()
def __del__(self):
logger.info(f"GGUFVisionGenerativeModel - cleaning up mtmd free on closing model instance")
if self.mtmd_ctx is not None:
self._libmtmd.mtmd_free(self.mtmd_ctx)
def load_model_for_inference(self, model_repo_path, model_card=None, **kwargs):
""" Loads and instantiates model along with other required objects. """
# needs to load both llama + clip models
if model_card:
self.model_card = model_card
# validate before loading
self.validate()
# load llama model
response = self._load_llama_model_for_inference(model_repo_path, model_card, **kwargs)
if not response:
logger.warning(f"GGUFVisionGenerativeModel - error loading llama backend model.")
# further triage and debug info steps ...
pass
# load clip model
response = self._load_clip_model_for_inference(model_repo_path, model_card, **kwargs)
if not response:
logger.warning(f"GGUFVisionGenerativeModel - error loading mtmd clip backend model.")
# further triage and debug info steps ...
pass
return self
def _load_clip_model_for_inference(self, model_repo_path, model_card=None, **kwargs):
""" Loads backend MTMD module along with instantiating CLIP Model and prepares associated context """
# load shared library
self._libmtmd = self.load_mtmd_shared_library()
self._libmtmd = add_libmtmd_ctypes_declarations(self._libmtmd)
# set up log (best effort) - catch and skip if any errors thrown
try:
self._libmtmd.mtmd_helper_log_set(mtmd_log_callback, ctypes.c_void_p(0))
except:
logger.info(f"GGUFVisionGenerativeModel - unable to set mtmd log")
ctx_params = self._libmtmd.mtmd_context_params_default()
ctx_params.use_gpu = True
ctx_params.print_timings = 0 # self.verbose
import multiprocessing
ctx_params.n_threads = max(multiprocessing.cpu_count() // 2, 1)
# deprecated
# ctx_params.verbosity = 0 # 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2
if not self.clip_file:
self.clip_file = "mmproj-F16.gguf"
self.clip_model_path = os.path.join(model_repo_path, self.clip_file)
# Initialize mtmd context
self.mtmd_ctx = self._libmtmd.mtmd_init_from_file(self.clip_model_path.encode(),
self._model.model, ctx_params)
if self.mtmd_ctx is None:
raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}")
# Check if vision is supported
if self._libmtmd.mtmd_support_vision(self.mtmd_ctx):
logger.info(f"GGUFVisionGenerativeModel - confirmed that model supports vision")
else:
logger.info(f"GGUFVisionGenerativeModel - model does not support vision - expect errors likely")
return True
def _load_llama_model_for_inference(self, model_repo_path, model_card=None, **kwargs):
""" Loads Llama model and sets context parameters """
# load shared library
self._lib = self._load_llama_cpp_shared_library()
self._lib = add_ctypes_declarations(self._lib)
if not GGUFConfigs().get_config("backend_initialized"):
# is this backend init required?
self._lib.llama_backend_init()
GGUFConfigs().set_config("backend_initialized", True)
self._lib.llama_log_set(llama_log_callback, ctypes.c_void_p(0))
self.model_params = self._lib.llama_model_default_params()
# update model params parameters
self.model_params.n_gpu_layers = 50
self.model_params.main_gpu = 0
self.model_params.vocab_only = False
self.model_params.use_mmap = True
self.model_params.use_mlock = False
if self.use_gpu:
# on darwin, keep at 0 - on win32 and linux - set to 50 by default (e.g., shift all model layers to GPU)
if sys.platform.lower() == "win32" or sys.platform.lower().startswith("linux"):
self.model_params.n_gpu_layers = GGUFConfigs().get_config("n_gpu_layers")
# update context parameters
self.context_params = self._lib.llama_context_default_params()
# sets minimum of 2048, but will extend if context_window is larger (e.g., 4096/8192+)
# self.context_params.n_ctx = max(2048, self.max_total_len)
self.context_params.n_ctx = 8192 # 2048
self.context_params.n_batch = self.n_batch
n_ubatch = 512
self.context_params.n_ubatch = min(self.n_batch, n_ubatch)
# note: handcrafting of thread allocation can sometimes help performance substantially
import multiprocessing
self.context_params.n_threads = max(multiprocessing.cpu_count() // 2, 1)
self.context_params.n_threads_batch = multiprocessing.cpu_count()
# self.context_params.rope_scaling_type = (LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED)
# self.context_params.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED
self.context_params.rope_freq_base = 0.0 # (rope_freq_base if rope_freq_base != 0.0 else 0)
self.context_params.type_k = 1
self.context_params.type_v = 1
self.context_params.offload_kqv = True
self.context_params.yarn_orig_ctx = 0
if model_card:
self.model_name = model_card["model_name"].split("/")[-1]
self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf",
self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf"
self.model_path = os.path.join(model_repo_path, self.gguf_file)
# loads and instantiates the key objects
self._model = _LlamaModel(self._lib, path_model=self.model_path, params=self.model_params)
self._ctx = _LlamaContext(self._lib, model=self._model, params=self.context_params)
self._batch = _LlamaBatch(self._lib, n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx)
self.vocab = self._lib.llama_model_get_vocab(self._model.model)
self._n_vocab = self.n_vocab()
self._n_ctx = self.n_ctx()
self._token_nl = self.token_nl()
self._token_eos = self.token_eos()
self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab)
self.input_ids = np.ndarray((self._n_ctx,), dtype=np.intc)
self.scores = np.ndarray((self._n_ctx, self._n_vocab), dtype=np.single)
self._sampler = self._init_sampler()
return True
def _load_llama_cpp_shared_library(self):
""" Loads llama_cpp shared library - checks if a custom lib path has been configured - otherwise,
it loads the llmware provided dynamic libraries based on the platform/system. """
# check first if custom_lib_path - expected to be full path to custom so/dylib file
custom_path = GGUFConfigs().get_config("custom_lib_path")
cdll_args = dict()
# add option to fall_back if CUDA driver can not be loaded correctly to CPU driver for that OS
fall_back_option = ""
if custom_path:
# point to custom llama.cpp backend libs
if os.path.exists(custom_path):
_lib_paths = [custom_path]
else:
raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib")
else:
_base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf")
_lib_paths = []
system_platform = sys.platform.lower()
# Determine the file extension based on the platform
if system_platform.startswith("linux"):
# three linux versions supported - linux_x86 and linux_cuda
machine = os.uname().machine.lower()
if machine == "aarch64" and self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"),
GGUFConfigs().get_config("linux_cuda")))
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"),
GGUFConfigs().get_config("linux_cuda")))
# will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86"))
else:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86")))
elif system_platform == "darwin":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"),
GGUFConfigs().get_config("mac_metal")))
elif sys.platform == "win32":
import platform
if platform.machine().lower() == "arm64":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"),
GGUFConfigs().get_config("windows_arm64")))
# windows cuda
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"),
GGUFConfigs().get_config("windows_cuda")))
# new - will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows"))
else:
# main case - windows x86
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows")))
else:
raise LLMWareException(message=f"No matching llama.cpp binary for platform - {system_platform}")
# Add the library directory to the DLL search path on Windows (if needed)
if sys.platform == "win32" and sys.version_info >= (3, 8):
os.add_dll_directory(str(_base_path))
# need to review
if "CUDA_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths:
logger.debug(f"Loading llama cpp backend - {_lib_path}")
if not os.path.exists(_lib_path):
if fall_back_option:
_lib_path = fall_back_option
if os.path.exists(_lib_path):
try:
return ctypes.cdll.LoadLibrary(str(_lib_path))
except Exception as e:
# if fail, and CUDA selected, then try to fall back to matching CPU version
if fall_back_option:
try:
logger.warning("Not successful loading preferred lib so reverting to fallback lib.")
return ctypes.cdll.LoadLibrary(str(_lib_path))
except:
# if fall-back fails
raise GGUFLibNotLoadedException("llama_cpp_backend",
sys.platform.lower(),
self.use_gpu,
_lib_path,
custom_path)
else:
raise GGUFLibNotLoadedException("llama_cpp_backend" ,sys.platform.lower(),
self.use_gpu, _lib_path, custom_path)
# if not loaded
raise LLMWareException(message=f"GGUFGenerativeModel - attempting to load llama cpp backend lib - "
f"Llama cpp backend not found.")
def _init_mtmd_context(self, llama_model):
"""Initialize mtmd context with the llama model."""
self.mtmd_ctx = None
# Get default parameters
ctx_params = self._libmtmd.mtmd_context_params_default()
ctx_params.use_gpu = True # todo: expose as configuration option directly
ctx_params.print_timings = self.verbose
ctx_params.n_threads = max(multiprocessing.cpu_count() // 2, 1)
# deprecated/removing
# ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2
# Initialize mtmd context
self.mtmd_ctx = self._libmtmd.mtmd_init_from_file(self.clip_model_path.encode(),
llama_model.model,
ctx_params)
if self.mtmd_ctx is None:
raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}")
# Check if vision is supported
if not self._libmtmd.mtmd_support_vision(self.mtmd_ctx):
raise ValueError("Vision is not supported by this model")
return True
def mtmd_free(self):
""" Deletes MTMD context """
if self.mtmd_ctx is not None:
self._mtmd_cpp.mtmd_free(self.mtmd_ctx)
self.mtmd_ctx = None
return True
def load_mtmd_shared_library(self):
"""Platform independent shared library loader for mtmd lib backend """
# providing several backends packaged within llmware for the following:
# "windows_mtmd": "mtmd.dll",
# "mac_metal_mtmd": "libmtmd.dylib",
# "linux_x86_mtmd": "libmtmd.so",
# "linux_cuda_mtmd": "libmtmd.so",
# "windows_arm64_mtmd": "mtmd.dll",
# "windows_cuda_mtmd": "mtmd.dll",
# check first if custom_lib_path - expected to be full path to custom so/dylib file
custom_path = GGUFConfigs().get_config("custom_lib_path")
cdll_args = dict()
fall_back_option = ""
if custom_path:
if os.path.exists(custom_path):
_lib_paths = [custom_path]
else:
raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib")
else:
_base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf")
_lib_paths = []
system_platform = sys.platform.lower()
# Determine the file extension based on the platform
if system_platform.startswith("linux"):
# three linux versions supported - linux_x86 and linux_cuda
machine = os.uname().machine.lower()
if machine == "aarch64" and self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"),
GGUFConfigs().get_config("linux_cuda_mtmd")))
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"),
GGUFConfigs().get_config("linux_cuda_mtmd")))
# will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86_mtmd"))
else:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"),
GGUFConfigs().get_config("linux_x86_mtmd")))
elif system_platform == "darwin":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"),
GGUFConfigs().get_config("mac_metal_mtmd")))
elif sys.platform == "win32":
import platform
if platform.machine().lower() == "arm64":
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"),
GGUFConfigs().get_config("windows_arm64_mtmd")))
# windows cuda
elif self.use_gpu:
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"),
GGUFConfigs().get_config("windows_cuda_mtmd")))
# new - will try to use x86 as fallback
fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows_mtmd"))
else:
# main case - windows x86
_lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"),
GGUFConfigs().get_config("windows_mtmd")))
else:
raise LLMWareException(message=f"No matching mtmd binary for platform - {system_platform}")
# Add the library directory to the DLL search path on Windows (if needed)
if sys.platform == "win32" and sys.version_info >= (3, 8):
os.add_dll_directory(str(_base_path))
# need to review
if "CUDA_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
# Try to load the shared library, handling potential errors
for _lib_path in _lib_paths:
logger.debug(f"Loading mtmd backend - {_lib_path}")
if not os.path.exists(_lib_path):
if fall_back_option:
_lib_path = fall_back_option
if os.path.exists(_lib_path):
try:
return ctypes.cdll.LoadLibrary(str(_lib_path))
except Exception as e:
# if fail, and CUDA selected, then try to fall back to matching CPU version
if fall_back_option:
try:
logger.warning("Not successful loading preferred lib so reverting to fallback lib.")
return ctypes.cdll.LoadLibrary(str(_lib_path))
except:
# if fall-back fails
raise GGUFLibNotLoadedException("mtmd_backend",
sys.platform.lower(),
self.use_gpu,
_lib_path,
custom_path)
else:
raise GGUFLibNotLoadedException("mtmd_backend", sys.platform.lower(),
self.use_gpu, _lib_path, custom_path)
# if not loaded
raise LLMWareException(message=f"GGUFVisionGenerativeModel - attempting to load mtmd backend lib - "
f"mtmd backend not found.")
def image_to_base64_data_uri(self, file_path):
""" Image handling utility """
import base64
with open(file_path, "rb") as img_file:
base64_data = base64.b64encode(img_file.read()).decode('utf-8')
return f"data:image/jpg;base64,{base64_data}"
def _create_bitmap_from_bytes(self, image_bytes: bytes):
"""Create mtmd_bitmap from image bytes."""
if self.mtmd_ctx is None:
raise ValueError("mtmd context not initialized")
bitmap = self._libmtmd.mtmd_helper_bitmap_init_from_buf(
self.mtmd_ctx,
(ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)),
len(image_bytes)
)
if bitmap is None:
raise ValueError("Failed to create bitmap from image bytes")
return bitmap
def prepare_image_prompt(self, prompt, image_path):
""" Main entry point for building image encodings and merging with token encodings to prepare
prompt for generative decoder model """
data_uri = self.image_to_base64_data_uri(image_path)
import base64
image_bytes = base64.b64decode(data_uri.split(",")[1])
bitmap = self._create_bitmap_from_bytes(image_bytes)
bitmaps = []
bitmap_cleanup = []
bitmaps.append(bitmap)
bitmap_cleanup.append(bitmap)
# Create input text structure
input_text = mtmd_input_text()
input_text.text = prompt.encode('utf-8')
input_text.add_special = True
input_text.parse_special = True
# Create input chunks
chunks = self._libmtmd.mtmd_input_chunks_init()
if chunks is None:
raise ValueError("Failed to create input chunks")
bitmap_array = (mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps)
result = self._libmtmd.mtmd_tokenize(
self.mtmd_ctx,
chunks,
ctypes.byref(input_text),
bitmap_array,
len(bitmaps)
)
if result != 0:
raise ValueError(f"Failed to tokenize input: error code {result}")
# Reset llama context
self.reset()
memory = self._lib.llama_get_memory(self._ctx.ctx)
self._lib.llama_memory_clear(memory, True)
# Process each chunk
n_past = llama_pos(0)
n_chunks = self._libmtmd.mtmd_input_chunks_size(chunks)
for i in range(n_chunks):
chunk = self._libmtmd.mtmd_input_chunks_get(chunks, i)
if chunk is None:
continue
chunk_type = self._libmtmd.mtmd_input_chunk_get_type(chunk)
if chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT:
# Handle text chunk
n_tokens_out = ctypes.c_size_t()
tokens_ptr = self._libmtmd.mtmd_input_chunk_get_tokens_text(
chunk, ctypes.byref(n_tokens_out)
)
if tokens_ptr and n_tokens_out.value > 0:
# Convert ctypes array to Python list
tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)]
if self.n_tokens + len(tokens) > self.n_ctx():
raise ValueError(
f"Prompt is larger than n_ctx: {self.n_tokens + len(tokens)} > {self.n_ctx()}"
)
self.eval(tokens)
elif chunk_type in [MTMD_INPUT_CHUNK_TYPE_IMAGE,
MTMD_INPUT_CHUNK_TYPE_AUDIO]:
chunk_n_tokens = self._libmtmd.mtmd_input_chunk_get_n_tokens(chunk)
if self.n_tokens + chunk_n_tokens > self.n_ctx():
raise ValueError(
f"Prompt is larger than n_ctx: {self.n_tokens + chunk_n_tokens} > {self.n_ctx()}"
)
new_n_past = llama_pos(0)
result = self._libmtmd.mtmd_helper_eval_chunk_single(
self.mtmd_ctx,
self._ctx.ctx,
chunk,
llama_pos(self.n_tokens),
llama_seq_id(0),
self.n_batch,
False, # logits_last
ctypes.byref(new_n_past)
)
if result != 0:
raise ValueError(f"Failed to evaluate chunk: error code {result}")
self.n_tokens = new_n_past.value
prompt = self.input_ids[: self.n_tokens].tolist()
self._libmtmd.mtmd_input_chunks_free(chunks)
state_size = self._lib.llama_state_get_size(self._ctx.ctx)
return prompt
def _init_sampler(self):
""" Initializes and sets up the llama cpp backend sampler """
# create sampler
# default params are struct
params = llama_sampler_chain_params()
self._sampler = self._lib.llama_sampler_chain_init(params)
temp = 0.0
# todo: expose more sampling options
if temp < 0.0:
# sampler.add_softmax()
self._lib.llama_sampler_chain_add(self._sampler, self._lib.llama_sampler_init_softmax())
# sampler.add_dist(self._seed)
elif temp == 0.0:
# sampler.add_greedy()
greedy_sampler = self._lib.llama_sampler_init_greedy()
self._lib.llama_sampler_chain_add(self._sampler, greedy_sampler)
return self._sampler
def _inference(self, prompt):
""" Tokenizes the prompt and executes generation loop. """
# self._sampler = self._init_sampler()
t0 = time.time()
completion_tokens = [] if len(prompt) > 0 else [self.token_bos()]
prompt_tokens = (
(
self.tokenize(prompt.encode("utf-8"), special=True)
if prompt != ""
else [self.token_bos()]
)
if isinstance(prompt, str)
else prompt
)
# confirm that input is smaller than context_window
input_len = len(prompt_tokens)
context_window = self.n_ctx()
if input_len > context_window:
logger.warning("GGUFCLIPGenerativeModel - input is too long for model context window - "
"truncating")
min_output_len = 10
prompt_tokens = prompt_tokens[0:context_window - min_output_len]
input_len = len(prompt_tokens)
text = b""
# first token capture starts here
get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed")
token_counter = 0
t_gen_start = time.time()
first_token_processing_time = -1.0
for token in self.generate(prompt_tokens):
# first token capture
if get_first_token_speed:
if token_counter == 0:
first_token_processing_time = time.time() - t_gen_start
token_counter += 1
# first token capture ends here
if self.get_logits:
self.register_top_logits()
self.output_tokens.append(token)
if token == self._token_eos:
text = self.detokenize(completion_tokens)
break
completion_tokens.append(token)
# stop at max output len
if len(completion_tokens) >= self.max_output_len:
text = self.detokenize(completion_tokens)
break
# stop if combined input + output at context window size
if (input_len + len(completion_tokens)) >= context_window:
text = self.detokenize(completion_tokens)
break
text_str = text.decode("utf-8", errors="ignore")
# post-processing clean-up - stop at endoftext
eot = text_str.find("<|endoftext|>")
if eot > -1:
text_str = text_str[:eot]
# new post-processing clean-up - stop at </s>
eots = text_str.find("</s>")
if eots > -1:
text_str = text_str[:eots]
# post-processing clean-up - start after bot wrapper
bot = text_str.find("<bot>:")
if bot > -1:
text_str = text_str[bot + len("<bot>:"):]
# new post-processing cleanup - skip repeating starting <s>
boss = text_str.find("<s>")
if boss > -1:
text_str = text_str[boss + len("<s>"):]
# end - post-processing
if get_first_token_speed:
output = {"llm_response": text_str,
"usage": {"input": len(prompt_tokens), "output": len(completion_tokens),
"total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens",
"processing_time": time.time() - t0,
"first_token_processing_time": first_token_processing_time}}
else:
output = {"llm_response": text_str,
"usage": {"input": len(prompt_tokens), "output": len(completion_tokens),
"total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens",
"processing_time": time.time() - t0}}
if self.get_logits:
output.update({"logits": self.logits_record, "output_tokens": self.output_tokens})
return output
def sample_gguf(self, idx=None):
""" Adapted to sample_gguf to avoid potential name space conflicts. """
# assert self.n_tokens > 0
tmp_sampler = False
if self._sampler is None:
tmp_sampler = True
self._sampler = self._init_sampler()
ridx = idx - self.n_tokens if idx is not None else -1
assert self.ctx is not None
token = self._lib.llama_sampler_sample(self._sampler, self._ctx.ctx, ridx)
# token = int(self.logits_record[-1][0][0])
if tmp_sampler:
self._sampler = None
return token
def generate(self, tokens, reset=True):
""" Generator that samples the model and yields tokens until stopped. """
# test
# Check for kv cache prefix match
if reset and self.n_tokens > 0:
longest_prefix = 0
for a, b in zip(self._input_ids, tokens[:-1]):
if a == b:
longest_prefix += 1
else:
break
if longest_prefix > 0:
reset = False
tokens = tokens[longest_prefix:]
self.n_tokens = longest_prefix
# Reset the model state
# reset = False
if reset:
self.reset()
sample_idx = self.n_tokens + len(tokens) - 1
tokens = list(tokens)
tokens_created = 0
input_start_len = len(tokens)
memory = self._ctx.memory
# Eval and sample
while True:
self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1)
for i in range(0, len(tokens), self.n_batch):
batch = tokens[i: min(len(tokens), i + self.n_batch)]
n_past = self.n_tokens
n_tokens = len(batch)
self._batch.set_batch(batch=batch, n_past=n_past, logits_all=self._logits_all)
return_code = self._lib.llama_decode(self._ctx.ctx, self._batch.batch)
# TODO: add better error handling if return_code 1 - usually overflow of ctx
if return_code != 0:
raise RuntimeError(f"llama_decode call returned {return_code} - in most cases, this "
f"is due to exceeding the maximum context window.")
self.input_ids[n_past: n_past + n_tokens] = batch
rows = n_tokens
cols = self._n_vocab
offset = (0 if self._logits_all else n_tokens - 1)
if self._logits_all:
rows = n_tokens
cols = self._n_vocab
logits = np.ctypeslib.as_array(
self._ctx.get_logits(), shape=(rows * cols,))
self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits
self.n_tokens += n_tokens
# leaving hard-coded off for now (improves performance)
# self.register_top_logits()
while sample_idx < self.n_tokens:
logits = self._scores[-1, :]
self.prev = list(self.eval_tokens)
# sample to generate token from logits
token = self.sample_gguf(idx=sample_idx) # (logits_array=logits)
self.accept(id=id, apply_grammar=None)
tokens_created += 1
sample_idx += 1
tokens_or_none = yield token
tokens.clear()
tokens.append(token)
if tokens_or_none is not None:
tokens.extend(tokens_or_none)
if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]:
self.n_tokens = sample_idx
self._lib.llama_memory_seq_rm(self._lib.llama_get_memory(self._ctx.ctx), -1, self.n_tokens, -1)
break
if tokens_created > self.max_output_len:
logger.info("GGUFVisionGenerativeModel - stopping generation loop - reached limit of "
"max output len")
break
def tokenize(self, text, add_bos=True, special=False):
""" Tokenizes text. """
n_ctx = self.n_ctx_train()
tokens = (ctypes.c_int32 * n_ctx)()
# change from self._model.model
n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_ctx, add_bos, special)
if n_tokens < 0:
n_tokens = abs(n_tokens)
tokens = (ctypes.c_int32 * n_tokens)()
n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_tokens, add_bos, special)
if n_tokens < 0:
raise RuntimeError(f"GGUFVisionGenerativeModel - tokenization error - "
f"{text} - n_tokens={n_tokens}")
return list(tokens[:n_tokens])
def detokenize(self, tokens, special: bool = False) -> bytes:
output = b""
size = 32
buffer = (ctypes.c_char * size)()
for token in tokens:
n = self._lib.llama_token_to_piece(
# replace: self.model
self.vocab, llama_token(token), buffer, size, 0, special
)
assert n <= size
output += bytes(buffer[:n])
# following llama_cpp_python on below ...
# NOTE: Llama1 models automatically added a space at the start of the prompt
# this line removes a leading space if the first token is a beginning of sentence token
return (
output[1:]
if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" "
else output
)
def accept(self, id, apply_grammar):
""" Formal step post sampling that 'accepts' and adds the token id to the running generation. """
if apply_grammar and self.grammar is not None:
self._lib.llama_grammar_accept_token(self._ctx.ctx, self.grammar.grammar, id)
self.prev.append(id)
def register_top_logits(self):
""" Gets the top logits and keeps a running log for output analysis. """
# TODO: there is issue with first logit computation - not corresponding to first token
logit_pointer = self._lib.llama_get_logits(self._ctx.ctx)
logit_size = self.n_vocab()
logit_array = np.zeros(logit_size)
for x in range(0, logit_size):
logit_array[x] = logit_pointer[x]
sm = np.exp(logit_array) / sum(np.exp(logit_array))
sm_sorted = np.sort(sm)
sm_args_sorted = np.argsort(sm)
top_logits = []
for x in range(0, self.top_logit_count):
# experiment - try rounding the float number
pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3))
top_logits.append(pair)
# print("--test: logits - ", x, top_logits)
self.logits_record.append(top_logits)
return top_logits
def set_api_key(self, api_key, env_var="USER_MANAGED_GGUF_API_KEY"):
""" Sets API key - generally not used in GGUF models. """
# set api_key
os.environ[env_var] = api_key
logger.info("added and stored GGUF api_key in environmental variable- %s", env_var)
return self
def _get_api_key(self, env_var="USER_MANAGED_GGUF_API_KEY"):
""" Gets API key - generally not used in GGUF models. """
self.api_key = os.environ.get(env_var)
if not self.api_key:
logger.warning("_get_api_key could not successfully retrieve value from: %s ", env_var)
return self.api_key
@property
def ctx(self):
return self._ctx.ctx
@property
def model(self):
return self._model.model
@property
def _input_ids(self):
return self.input_ids[: self.n_tokens]
@property
def _scores(self):
return self.scores[: self.n_tokens, :]
@property
def eval_tokens(self):
return deque(self.input_ids[: self.n_tokens].tolist(), maxlen=self._n_ctx)
def eval(self, tokens):
"""Evaluate a list of tokens.
Args:
tokens: The list of tokens to evaluate.
"""
memory = self._ctx.memory
self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1)
for i in range(0, len(tokens), self.n_batch):
batch = tokens[i: min(len(tokens), i + self.n_batch)]
n_past = self.n_tokens
n_tokens = len(batch)
self._batch.set_batch(
batch=batch, n_past=n_past, logits_all=self._logits_all
)
self._lib.llama_decode(self._ctx.ctx, self._batch.batch)
# Save tokens
self.input_ids[n_past: n_past + n_tokens] = batch
# Save logits
if self._logits_all:
rows = n_tokens
cols = self._n_vocab
logits = np.ctypeslib.as_array(
self._ctx.get_logits(), shape=(rows * cols,)
)
self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits
else:
pass
# Update n_tokens
self.n_tokens += n_tokens
@property
def eval_logits(self):
return deque(
self.scores[: self.n_tokens, :].tolist(),
maxlen=self._n_ctx if self._logits_all else 1,
)
def reset(self):
self.n_tokens = 0
def n_ctx(self):
return self._lib.llama_n_ctx(self._ctx.ctx)
def n_ctx_train(self):
return self._lib.llama_n_ctx_train(self._model.model)
def n_vocab(self):
# llama_model_get_vocab(model)
n_vocab = self._lib.llama_n_vocab(self._lib.llama_model_get_vocab(self._model.model))
return n_vocab
def token_eos(self):
# return self._lib.llama_token_eos(self._model.model)
eos = self._lib.llama_token_eos(self.vocab)
return eos
def token_bos(self):
# return self._lib.llama_token_bos(self._model.model)
bos = self._lib.llama_token_bos(self.vocab)
return bos
def token_nl(self):
token_nl = self._lib.llama_token_nl(self._lib.llama_model_get_vocab(self._model.model))
# return self._lib.llama_token_nl(self._model.model)
return token_nl
def unload_model(self):
""" Unloads a model to release memory """
# note: removing pointer seems to safely remove from Python reference tracking
self._batch = None
self._ctx = None
self._model = None
return 0
def inference(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None, get_logits=False, disable_eos=False):
""" Main method for inference generation. """
logger.info("GGUFVisionGenerativeModel - Starting generation inference")
time_start = time.time()
media_marker = self._libmtmd.mtmd_default_marker().decode('utf-8')
text = "\n" + str(media_marker) + prompt
self.prompt = text
prompt = self.prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# update default handling for no add_prompt_engineering
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# start with clean logits_record and output_tokens for each function call
self.logits_record = []
self.output_tokens = []
if get_logits:
self.get_logits = get_logits
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# preview before generation
# self.preview()
# prompt = prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
prompt = prompt_final + self.trailing_space
# prepare embedded image prompt with fully templated prompt
prompt_tokens = self.prepare_image_prompt(prompt, image_path)
# output_response = self._inference(text_prompt)
# starts _inference here
completion_tokens = [] if len(prompt_tokens) > 0 else [self.token_bos()]
# todo: safety checks to confirm that input is smaller than context_window
input_len = len(prompt_tokens)
context_window = self.n_ctx()
text = b""
token_list = []
token_counter = 0
text_output = ""
for token in self.generate(prompt_tokens):
completion_tokens.append(token)
if not disable_eos:
if token == self._token_eos:
break
if len(completion_tokens) > self.max_output_len:
break
# stop if combined input + output at context window size
if (input_len + len(completion_tokens)) >= context_window:
break
new_token = self.detokenize([token]).decode('utf-8', errors='ignore')
text_output += new_token
token_counter += 1
# text_str = text_output.decode("utf-8", errors="ignore")
usage = {"input": input_len,
"output": token_counter,
"total": input_len + token_counter,
"metric": "tokens",
"processing_time": time.time() - time_start}
response = {"llm_response": text_output, "usage": usage}
self.register()
return response
def stream(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None,
inference_dict=None,
get_logits=False, disable_eos=False):
""" Main method for text streaming generation. Returns a generator function that yields one
token at a time for real-time streaming to console or UI. """
logger.info("GGUFVisionGenerativeModel - Starting generation stream")
media_marker = self._libmtmd.mtmd_default_marker().decode('utf-8')
text = "\n" + str(media_marker) + prompt
self.prompt = text
prompt = self.prompt
if add_context:
self.add_context = add_context
if add_prompt_engineering:
self.add_prompt_engineering = add_prompt_engineering
# update default handling for no add_prompt_engineering
if not self.add_prompt_engineering:
if self.add_context:
self.add_prompt_engineering = "default_with_context"
else:
self.add_prompt_engineering = "default_no_context"
# start with clean logits_record and output_tokens for each function call
self.logits_record = []
self.output_tokens = []
if get_logits:
self.get_logits = get_logits
if inference_dict:
if "temperature" in inference_dict:
self.temperature = inference_dict["temperature"]
if "max_tokens" in inference_dict:
self.target_requested_output_tokens = inference_dict["max_tokens"]
# preview before generation
# self.preview()
# prompt = prompt
if self.add_prompt_engineering:
prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict)
prompt_final = prompt_enriched
# most models perform better with no trailing space or line-break at the end of prompt
# -- in most cases, the trailing space will be ""
# -- yi model prefers a trailing "\n"
# -- keep as parameterized option to maximize generation performance
# -- can be passed either thru model_card or model config from HF
prompt = prompt_final + self.trailing_space
# prepare embedded image prompt with fully templated prompt
prompt_tokens = self.prepare_image_prompt(prompt, image_path)
# output_response = self._inference(text_prompt)
# starts _inference here
completion_tokens = [] if len(prompt_tokens) > 0 else [self.token_bos()]
#todo: safety checks to confirm that input is smaller than context_window
input_len = len(prompt_tokens)
context_window = self.n_ctx()
text = b""
# disable_eos = True
token_list = []
for token in self.generate(prompt_tokens):
completion_tokens.append(token)
if not disable_eos:
if token == self._token_eos:
break
if len(completion_tokens) > self.max_output_len:
break
# stop if combined input + output at context window size
if (input_len + len(completion_tokens)) >= context_window:
break
new_token = self.detokenize([token]).decode('utf-8', errors='ignore')
yield new_token
text_str = text.decode("utf-8", errors="ignore")
# turned off
self.register()
return text_str
def function_call(self, context, function=None, params=None,get_logits=True,temperature=-99.0,max_output=None):
""" Not implemented for this model class. """
return True
def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="",
function=None, endpoint_base=None, api_key=None, get_logits=False):
""" Not implemented for this model class """
return True
def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False):
""" Not implemented for this model class """
return True
Reference in New Issue View Git Blame Copy Permalink