chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,218 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any, ClassVar
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.connectors.ai.onnx.onnx_gen_ai_prompt_execution_settings import OnnxGenAIPromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.onnx.onnx_gen_ai_settings import OnnxGenAISettings
|
||||
from semantic_kernel.connectors.ai.onnx.services.onnx_gen_ai_completion_base import OnnxGenAICompletionBase
|
||||
from semantic_kernel.connectors.ai.onnx.utils import ONNXTemplate, apply_template
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents import (
|
||||
AudioContent,
|
||||
ChatHistory,
|
||||
ChatMessageContent,
|
||||
ImageContent,
|
||||
StreamingChatMessageContent,
|
||||
TextContent,
|
||||
)
|
||||
from semantic_kernel.contents.utils.author_role import AuthorRole
|
||||
from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidExecutionSettingsError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@experimental
|
||||
class OnnxGenAIChatCompletion(ChatCompletionClientBase, OnnxGenAICompletionBase):
|
||||
"""OnnxGenAI text completion service."""
|
||||
|
||||
template: ONNXTemplate | None
|
||||
SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template: ONNXTemplate | None = None,
|
||||
ai_model_path: str | None = None,
|
||||
ai_model_id: str | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OnnxGenAITextCompletion class.
|
||||
|
||||
Args:
|
||||
template : The chat template configuration.
|
||||
ai_model_path : Local path to the ONNX model Folder.
|
||||
ai_model_id : The ID of the AI model. Defaults to None.
|
||||
env_file_path : Use the environment settings file as a fallback
|
||||
to environment variables.
|
||||
env_file_encoding : The encoding of the environment settings file.
|
||||
kwargs : Additional arguments.
|
||||
"""
|
||||
try:
|
||||
settings = OnnxGenAISettings(
|
||||
chat_model_folder=ai_model_path,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as e:
|
||||
raise ServiceInitializationError(f"Error creating OnnxGenAISettings: {e}") from e
|
||||
|
||||
if settings.chat_model_folder is None:
|
||||
raise ServiceInitializationError(
|
||||
"AI model path is not provided. Please provide the 'ai_model_path' parameter in the constructor. "
|
||||
"OR set the 'ONNX_GEN_AI_CHAT_MODEL_FOLDER' environment variable."
|
||||
)
|
||||
|
||||
if ai_model_id is None:
|
||||
ai_model_id = settings.chat_model_folder
|
||||
|
||||
super().__init__(ai_model_id=ai_model_id, ai_model_path=settings.chat_model_folder, template=template, **kwargs)
|
||||
|
||||
if self.enable_multi_modality and template is None:
|
||||
raise ServiceInitializationError(
|
||||
"When using a multi-modal model, a template must be specified."
|
||||
" Please provide a ONNXTemplate in the constructor."
|
||||
)
|
||||
|
||||
@override
|
||||
async def _inner_get_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> list["ChatMessageContent"]:
|
||||
"""Create chat message contents, in the number specified by the settings.
|
||||
|
||||
Args:
|
||||
chat_history : A list of chats in a chat_history object, that can be
|
||||
rendered into messages from system, user, assistant and tools.
|
||||
settings : Settings for the request.
|
||||
|
||||
Returns:
|
||||
A list of chat message contents representing the response(s) from the LLM.
|
||||
"""
|
||||
if not isinstance(settings, OnnxGenAIPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OnnxGenAIPromptExecutionSettings) # nosec
|
||||
prompt = self._prepare_chat_history_for_request(chat_history)
|
||||
images = self._get_images_from_history(chat_history)
|
||||
audios = self._get_audios_from_history(chat_history)
|
||||
choices = await self._generate_next_token(prompt, settings, images=images, audios=audios)
|
||||
return [self._create_chat_message_content(choice) for choice in choices]
|
||||
|
||||
@override
|
||||
async def _inner_get_streaming_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
function_invoke_attempt: int = 0,
|
||||
) -> AsyncGenerator[list["StreamingChatMessageContent"], Any]:
|
||||
"""Create streaming chat message contents, in the number specified by the settings.
|
||||
|
||||
Args:
|
||||
chat_history : A list of chat chat_history, that can be rendered into a
|
||||
set of chat_history, from system, user, assistant and function.
|
||||
settings : Settings for the request.
|
||||
function_invoke_attempt : The function invoke attempt.
|
||||
|
||||
Yields:
|
||||
A stream representing the response(s) from the LLM.
|
||||
"""
|
||||
if not isinstance(settings, OnnxGenAIPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OnnxGenAIPromptExecutionSettings) # nosec
|
||||
prompt = self._prepare_chat_history_for_request(chat_history)
|
||||
images = self._get_images_from_history(chat_history)
|
||||
audios = self._get_audios_from_history(chat_history)
|
||||
async for chunk in self._generate_next_token_async(prompt, settings, images=images, audios=audios):
|
||||
yield [
|
||||
self._create_streaming_chat_message_content(choice_index, new_token, function_invoke_attempt)
|
||||
for choice_index, new_token in enumerate(chunk)
|
||||
]
|
||||
|
||||
def _create_chat_message_content(self, choice: str) -> ChatMessageContent:
|
||||
return ChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
items=[
|
||||
TextContent(
|
||||
text=choice,
|
||||
ai_model_id=self.ai_model_id,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
def _create_streaming_chat_message_content(
|
||||
self, choice_index: int, choice: str, function_invoke_attempt: int
|
||||
) -> StreamingChatMessageContent:
|
||||
return StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
choice_index=choice_index,
|
||||
content=choice,
|
||||
ai_model_id=self.ai_model_id,
|
||||
function_invoke_attempt=function_invoke_attempt,
|
||||
)
|
||||
|
||||
@override
|
||||
def _prepare_chat_history_for_request(
|
||||
self, chat_history: ChatHistory, role_key: str = "role", content_key: str = "content"
|
||||
) -> Any:
|
||||
if self.template:
|
||||
return apply_template(chat_history, self.template)
|
||||
return self.tokenizer.apply_chat_template(
|
||||
json.dumps(self._chat_messages_to_dicts(chat_history)),
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
def _chat_messages_to_dicts(self, chat_history: "ChatHistory") -> list[dict[str, Any]]:
|
||||
return [
|
||||
message.to_dict(role_key="role", content_key="content")
|
||||
for message in chat_history.messages
|
||||
if isinstance(message, ChatMessageContent)
|
||||
]
|
||||
|
||||
def _get_images_from_history(self, chat_history: "ChatHistory") -> list[ImageContent] | None:
|
||||
images = []
|
||||
for message in chat_history.messages:
|
||||
for image in message.items:
|
||||
if isinstance(image, ImageContent):
|
||||
if not self.enable_multi_modality:
|
||||
raise ServiceInvalidExecutionSettingsError("The model does not support multi-modality")
|
||||
if image.uri:
|
||||
images.append(image)
|
||||
else:
|
||||
raise ServiceInvalidExecutionSettingsError(
|
||||
"Image Content URI needs to be set, because onnx can only work with file paths"
|
||||
)
|
||||
return images if len(images) else None
|
||||
|
||||
def _get_audios_from_history(self, chat_history: "ChatHistory") -> list[AudioContent] | None:
|
||||
audios = []
|
||||
for message in chat_history.messages:
|
||||
for audio in message.items:
|
||||
if isinstance(audio, AudioContent):
|
||||
if not self.enable_multi_modality:
|
||||
raise ServiceInvalidExecutionSettingsError("The model does not support multi-modality")
|
||||
if audio.uri:
|
||||
audios.append(audio)
|
||||
else:
|
||||
raise ServiceInvalidExecutionSettingsError(
|
||||
"Audio Content URI needs to be set, because onnx can only work with file paths"
|
||||
)
|
||||
return audios if len(audios) else None
|
||||
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
"""Create a request settings object."""
|
||||
return OnnxGenAIPromptExecutionSettings
|
||||
@@ -0,0 +1,109 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
|
||||
from semantic_kernel.connectors.ai.onnx.onnx_gen_ai_prompt_execution_settings import OnnxGenAIPromptExecutionSettings
|
||||
from semantic_kernel.contents import AudioContent, ImageContent
|
||||
from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidResponseError
|
||||
from semantic_kernel.kernel_pydantic import KernelBaseModel
|
||||
|
||||
try:
|
||||
import onnxruntime_genai as OnnxRuntimeGenAi
|
||||
|
||||
ready = True
|
||||
except ImportError:
|
||||
ready = False
|
||||
|
||||
|
||||
class OnnxGenAICompletionBase(KernelBaseModel):
|
||||
"""Base class for OnnxGenAI Completion services."""
|
||||
|
||||
model: Any
|
||||
tokenizer: Any
|
||||
tokenizer_stream: Any
|
||||
enable_multi_modality: bool = False
|
||||
|
||||
def __init__(self, ai_model_path: str, **kwargs) -> None:
|
||||
"""Creates a new instance of the OnnxGenAICompletionBase class, loads model & tokenizer.
|
||||
|
||||
Args:
|
||||
ai_model_path : Path to Onnx Model.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Raises:
|
||||
ServiceInitializationError: When model cannot be loaded
|
||||
"""
|
||||
if not ready:
|
||||
raise ImportError("onnxruntime-genai is not installed.")
|
||||
try:
|
||||
json_gen_ai_config = os.path.join(ai_model_path + "/genai_config.json")
|
||||
with open(json_gen_ai_config) as file:
|
||||
config: dict = json.load(file)
|
||||
enable_multi_modality = "vision" in config.get("model", {})
|
||||
model = OnnxRuntimeGenAi.Model(ai_model_path)
|
||||
if enable_multi_modality:
|
||||
tokenizer = model.create_multimodal_processor()
|
||||
else:
|
||||
tokenizer = OnnxRuntimeGenAi.Tokenizer(model)
|
||||
tokenizer_stream = tokenizer.create_stream()
|
||||
except Exception as ex:
|
||||
raise ServiceInitializationError("Failed to initialize OnnxCompletion service", ex) from ex
|
||||
|
||||
super().__init__(
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
tokenizer_stream=tokenizer_stream,
|
||||
enable_multi_modality=enable_multi_modality,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def _generate_next_token_async(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: OnnxGenAIPromptExecutionSettings,
|
||||
images: list[ImageContent] | None = None,
|
||||
audios: list[AudioContent] | None = None,
|
||||
) -> AsyncGenerator[list[str], Any]:
|
||||
try:
|
||||
params = OnnxRuntimeGenAi.GeneratorParams(self.model)
|
||||
params.set_search_options(**settings.prepare_settings_dict())
|
||||
generator = OnnxRuntimeGenAi.Generator(self.model, params)
|
||||
if not self.enable_multi_modality:
|
||||
input_tokens = self.tokenizer.encode(prompt)
|
||||
generator.append_tokens(input_tokens)
|
||||
else:
|
||||
# With the use of Pybind in ONNX there is currently no way to load images from bytes
|
||||
# We can only open images & audios from a file path currently
|
||||
if images is not None:
|
||||
images = OnnxRuntimeGenAi.Images.open(*[str(image.uri) for image in images])
|
||||
if audios is not None:
|
||||
audios = OnnxRuntimeGenAi.Audios.open(*[str(audio.uri) for audio in audios])
|
||||
input_tokens = self.tokenizer(prompt, images=images, audios=audios)
|
||||
generator.set_inputs(input_tokens)
|
||||
|
||||
while not generator.is_done():
|
||||
generator.generate_next_token()
|
||||
new_token_choices = [self.tokenizer_stream.decode(token) for token in generator.get_next_tokens()]
|
||||
yield new_token_choices
|
||||
del generator
|
||||
except Exception as ex:
|
||||
raise ServiceInvalidResponseError("Failed Inference with ONNX", ex) from ex
|
||||
|
||||
async def _generate_next_token(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: OnnxGenAIPromptExecutionSettings,
|
||||
images: list[ImageContent] | None = None,
|
||||
audios: list[AudioContent] | None = None,
|
||||
):
|
||||
token_choices: list[str] = []
|
||||
async for new_token_choice in self._generate_next_token_async(prompt, settings, images, audios=audios):
|
||||
# zip only works if the lists are the same length
|
||||
if len(token_choices) == 0:
|
||||
token_choices = new_token_choice
|
||||
else:
|
||||
token_choices = [old_token + new_token for old_token, new_token in zip(token_choices, new_token_choice)]
|
||||
return token_choices
|
||||
@@ -0,0 +1,133 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.onnx.onnx_gen_ai_prompt_execution_settings import OnnxGenAIPromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.onnx.onnx_gen_ai_settings import OnnxGenAISettings
|
||||
from semantic_kernel.connectors.ai.onnx.services.onnx_gen_ai_completion_base import OnnxGenAICompletionBase
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@experimental
|
||||
class OnnxGenAITextCompletion(TextCompletionClientBase, OnnxGenAICompletionBase):
|
||||
"""OnnxGenAI text completion service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_path: str | None = None,
|
||||
ai_model_id: str | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OnnxGenAITextCompletion class.
|
||||
|
||||
Args:
|
||||
ai_model_path : Local path to the ONNX model Folder.
|
||||
ai_model_id : The ID of the AI model. Defaults to None.
|
||||
env_file_path : Use the environment settings file as a fallback
|
||||
to environment variables.
|
||||
env_file_encoding : The encoding of the environment settings file.
|
||||
"""
|
||||
try:
|
||||
settings = OnnxGenAISettings(
|
||||
text_model_folder=ai_model_path,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as e:
|
||||
raise ServiceInitializationError(f"Invalid settings for OnnxGenAITextCompletion: {e}")
|
||||
|
||||
if settings.text_model_folder is None:
|
||||
raise ServiceInitializationError(
|
||||
"AI model path is not provided. Please provide the 'ai_model_path' parameter in the constructor. "
|
||||
"OR set the 'ONNX_GEN_AI_TEXT_MODEL_FOLDER' environment variable."
|
||||
)
|
||||
|
||||
if ai_model_id is None:
|
||||
ai_model_id = settings.text_model_folder
|
||||
|
||||
super().__init__(
|
||||
ai_model_id=ai_model_id,
|
||||
ai_model_path=settings.text_model_folder,
|
||||
)
|
||||
|
||||
@override
|
||||
async def _inner_get_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: PromptExecutionSettings,
|
||||
) -> list[TextContent]:
|
||||
"""This is the method that is called from the kernel to get a response from a text-optimized LLM.
|
||||
|
||||
Args:
|
||||
prompt : The prompt to send to the LLM.
|
||||
settings : Settings for the request.
|
||||
|
||||
Returns:
|
||||
List[TextContent]: A list of TextContent objects representing the response(s) from the LLM.
|
||||
"""
|
||||
if not isinstance(settings, OnnxGenAIPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OnnxGenAIPromptExecutionSettings) # nosec
|
||||
|
||||
choices = await self._generate_next_token(prompt, settings)
|
||||
return [
|
||||
TextContent(
|
||||
text=choice,
|
||||
ai_model_id=self.ai_model_id,
|
||||
)
|
||||
for choice in choices
|
||||
]
|
||||
|
||||
@override
|
||||
async def _inner_get_streaming_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: PromptExecutionSettings,
|
||||
) -> AsyncGenerator[list[StreamingTextContent], Any]:
|
||||
"""Streams a text completion using a Onnx model.
|
||||
|
||||
Note that this method does not support multiple responses.
|
||||
|
||||
Args:
|
||||
prompt : Prompt to complete.
|
||||
settings : Request settings.
|
||||
|
||||
Yields:
|
||||
List[StreamingTextContent]: List of StreamingTextContent objects.
|
||||
"""
|
||||
if not isinstance(settings, OnnxGenAIPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OnnxGenAIPromptExecutionSettings) # nosec
|
||||
|
||||
async for token_choice in self._generate_next_token_async(prompt, settings):
|
||||
yield [
|
||||
StreamingTextContent(
|
||||
choice_index=index, inner_content=new_token, text=new_token, ai_model_id=self.ai_model_id
|
||||
)
|
||||
for index, new_token in enumerate(token_choice)
|
||||
]
|
||||
|
||||
return
|
||||
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
"""Create a request settings object."""
|
||||
return OnnxGenAIPromptExecutionSettings
|
||||
Reference in New Issue
Block a user