chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,995 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
|
||||
|
||||
import numpy as np
|
||||
from aiohttp import ClientSession
|
||||
from aiortc import (
|
||||
MediaStreamTrack,
|
||||
RTCConfiguration,
|
||||
RTCDataChannel,
|
||||
RTCIceServer,
|
||||
RTCPeerConnection,
|
||||
RTCSessionDescription,
|
||||
)
|
||||
from av.audio.frame import AudioFrame
|
||||
from numpy import ndarray
|
||||
from openai._models import construct_type_unchecked
|
||||
from openai.resources.realtime.realtime import AsyncRealtimeConnection
|
||||
from openai.types.realtime import (
|
||||
ConversationItemCreateEvent,
|
||||
ConversationItemDeleteEvent,
|
||||
ConversationItemTruncateEvent,
|
||||
InputAudioBufferAppendEvent,
|
||||
InputAudioBufferClearEvent,
|
||||
InputAudioBufferCommitEvent,
|
||||
RealtimeClientEvent,
|
||||
RealtimeConversationItemFunctionCall,
|
||||
RealtimeConversationItemFunctionCallOutput,
|
||||
RealtimeConversationItemUserMessage,
|
||||
RealtimeResponseCreateParams,
|
||||
RealtimeServerEvent,
|
||||
ResponseCancelEvent,
|
||||
ResponseCreateEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
SessionUpdateEvent,
|
||||
)
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
|
||||
from semantic_kernel.connectors.ai.function_calling_utils import prepare_settings_for_function_calling
|
||||
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
|
||||
OpenAIRealtimeExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase
|
||||
from semantic_kernel.const import USER_AGENT
|
||||
from semantic_kernel.contents.audio_content import AudioContent
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.function_call_content import FunctionCallContent
|
||||
from semantic_kernel.contents.function_result_content import FunctionResultContent
|
||||
from semantic_kernel.contents.realtime_events import (
|
||||
RealtimeAudioEvent,
|
||||
RealtimeEvent,
|
||||
RealtimeEvents,
|
||||
RealtimeFunctionCallEvent,
|
||||
RealtimeFunctionResultEvent,
|
||||
RealtimeTextEvent,
|
||||
)
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.exceptions import ContentException
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT, prepend_semantic_kernel_to_user_agent
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiortc.mediastreams import MediaStreamTrack
|
||||
|
||||
from semantic_kernel.connectors.ai.function_choice_behavior import (
|
||||
FunctionCallChoiceConfiguration,
|
||||
FunctionChoiceType,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
|
||||
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
logger: logging.Logger = logging.getLogger("semantic_kernel.connectors.ai.open_ai.realtime")
|
||||
|
||||
|
||||
# region utils
|
||||
|
||||
|
||||
def update_settings_from_function_call_configuration(
|
||||
function_choice_configuration: "FunctionCallChoiceConfiguration",
|
||||
settings: "PromptExecutionSettings",
|
||||
type: "FunctionChoiceType",
|
||||
) -> None:
|
||||
"""Update the settings from a FunctionChoiceConfiguration."""
|
||||
if (
|
||||
function_choice_configuration.available_functions
|
||||
and hasattr(settings, "tool_choice")
|
||||
and hasattr(settings, "tools")
|
||||
):
|
||||
settings.tool_choice = type # type: ignore
|
||||
settings.tools = [ # type: ignore
|
||||
kernel_function_metadata_to_function_call_format(f)
|
||||
for f in function_choice_configuration.available_functions
|
||||
]
|
||||
|
||||
|
||||
def kernel_function_metadata_to_function_call_format(
|
||||
metadata: "KernelFunctionMetadata",
|
||||
) -> dict[str, Any]:
|
||||
"""Convert the kernel function metadata to function calling format.
|
||||
|
||||
Function calling in the realtime API, uses a slightly different format than the chat completion API.
|
||||
See https://platform.openai.com/docs/api-reference/realtime-sessions/create#realtime-sessions-create-tools
|
||||
for more details.
|
||||
|
||||
TLDR: there is no "function" key, and the function details are at the same level as "type".
|
||||
"""
|
||||
return {
|
||||
"type": "function",
|
||||
"name": metadata.fully_qualified_name,
|
||||
"description": metadata.description or "",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
param.name: param.schema_data for param in metadata.parameters if param.include_in_function_choices
|
||||
},
|
||||
"required": [p.name for p in metadata.parameters if p.is_required and p.include_in_function_choices],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# region constants
|
||||
|
||||
|
||||
@experimental
|
||||
class SendEvents(str, Enum):
|
||||
"""Events that can be sent."""
|
||||
|
||||
SESSION_UPDATE = "session.update"
|
||||
INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append"
|
||||
INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit"
|
||||
INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear"
|
||||
CONVERSATION_ITEM_CREATE = "conversation.item.create"
|
||||
CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate"
|
||||
CONVERSATION_ITEM_DELETE = "conversation.item.delete"
|
||||
RESPONSE_CREATE = "response.create"
|
||||
RESPONSE_CANCEL = "response.cancel"
|
||||
|
||||
|
||||
def _create_openai_realtime_client_event(event_type: SendEvents | str, **kwargs: Any) -> RealtimeClientEvent:
|
||||
"""Create an OpenAI Realtime client event from a event type and kwargs."""
|
||||
if isinstance(event_type, str):
|
||||
event_type = SendEvents(event_type)
|
||||
match event_type:
|
||||
case SendEvents.SESSION_UPDATE:
|
||||
if "session" not in kwargs:
|
||||
raise ContentException("Session is required for SessionUpdateEvent")
|
||||
session_dict = kwargs.pop("session")
|
||||
# Create proper RealtimeSessionCreateRequest with required type field for SDK validation
|
||||
# The OpenAI SDK will handle the proper serialization for the API
|
||||
from openai.types.realtime import RealtimeSessionCreateRequest
|
||||
|
||||
session_request = RealtimeSessionCreateRequest(type="realtime", **session_dict)
|
||||
return SessionUpdateEvent(
|
||||
type=event_type.value,
|
||||
session=session_request,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_APPEND:
|
||||
if "audio" not in kwargs:
|
||||
raise ContentException("Audio is required for InputAudioBufferAppendEvent")
|
||||
return InputAudioBufferAppendEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_COMMIT:
|
||||
return InputAudioBufferCommitEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_CLEAR:
|
||||
return InputAudioBufferClearEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.CONVERSATION_ITEM_CREATE:
|
||||
if "item" not in kwargs:
|
||||
raise ContentException("Item is required for ConversationItemCreateEvent")
|
||||
kwargs["type"] = event_type.value
|
||||
return ConversationItemCreateEvent(**kwargs)
|
||||
case SendEvents.CONVERSATION_ITEM_TRUNCATE:
|
||||
if "content_index" not in kwargs:
|
||||
kwargs["content_index"] = 0
|
||||
return ConversationItemTruncateEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.CONVERSATION_ITEM_DELETE:
|
||||
if "item_id" not in kwargs:
|
||||
raise ContentException("Item ID is required for ConversationItemDeleteEvent")
|
||||
return ConversationItemDeleteEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.RESPONSE_CREATE:
|
||||
if "response" in kwargs:
|
||||
response: RealtimeResponseCreateParams | None = RealtimeResponseCreateParams.model_validate(
|
||||
kwargs.pop("response")
|
||||
)
|
||||
else:
|
||||
response = None
|
||||
return ResponseCreateEvent(
|
||||
type=event_type.value,
|
||||
response=response,
|
||||
**kwargs,
|
||||
)
|
||||
case SendEvents.RESPONSE_CANCEL:
|
||||
return ResponseCancelEvent(
|
||||
type=event_type.value,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@experimental
|
||||
class ListenEvents(str, Enum):
|
||||
"""Events that can be listened to."""
|
||||
|
||||
ERROR = "error"
|
||||
SESSION_CREATED = "session.created"
|
||||
SESSION_UPDATED = "session.updated"
|
||||
CONVERSATION_CREATED = "conversation.created"
|
||||
INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed"
|
||||
INPUT_AUDIO_BUFFER_CLEARED = "input_audio_buffer.cleared"
|
||||
INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started"
|
||||
INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped"
|
||||
CONVERSATION_ITEM_CREATED = "conversation.item.created"
|
||||
CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED = "conversation.item.input_audio_transcription.completed"
|
||||
CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED = "conversation.item.input_audio_transcription.failed"
|
||||
CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated"
|
||||
CONVERSATION_ITEM_DELETED = "conversation.item.deleted"
|
||||
RESPONSE_CREATED = "response.created"
|
||||
RESPONSE_DONE = "response.done" # contains usage info -> log
|
||||
RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added"
|
||||
RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done"
|
||||
RESPONSE_CONTENT_PART_ADDED = "response.content_part.added"
|
||||
RESPONSE_CONTENT_PART_DONE = "response.content_part.done"
|
||||
RESPONSE_TEXT_DELTA = "response.output_text.delta"
|
||||
RESPONSE_TEXT_DONE = "response.output_text.done"
|
||||
RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.output_audio_transcript.delta"
|
||||
RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.output_audio_transcript.done"
|
||||
RESPONSE_AUDIO_DELTA = "response.output_audio.delta"
|
||||
RESPONSE_AUDIO_DONE = "response.output_audio.done"
|
||||
RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
|
||||
RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
|
||||
RATE_LIMITS_UPDATED = "rate_limits.updated"
|
||||
|
||||
|
||||
# region Base
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase):
|
||||
"""OpenAI Realtime service."""
|
||||
|
||||
SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True
|
||||
|
||||
_current_settings: PromptExecutionSettings | None = PrivateAttr(default=None)
|
||||
_call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Post init hook."""
|
||||
super().model_post_init(__context)
|
||||
if self.model_extra:
|
||||
if "kernel" in self.model_extra:
|
||||
self._kernel = self.model_extra["kernel"]
|
||||
if "plugins" in self.model_extra:
|
||||
self._add_plugin_to_kernel(self.model_extra["plugins"])
|
||||
if "settings" in self.model_extra:
|
||||
self._current_settings = self.model_extra["settings"]
|
||||
if "chat_history" in self.model_extra:
|
||||
self._chat_history = self.model_extra["chat_history"]
|
||||
|
||||
async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[RealtimeEvents, None]:
|
||||
"""Handle all events but audio delta.
|
||||
|
||||
Audio delta has to be handled by the implementation of the protocol as some
|
||||
protocols have different ways of handling audio.
|
||||
|
||||
We put all event in the output buffer, but after the interpreted one.
|
||||
so when dealing with them, make sure to check the type of the event, since they
|
||||
might be of different types.
|
||||
"""
|
||||
match event.type:
|
||||
case (
|
||||
ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value
|
||||
| "response.audio_transcript.delta"
|
||||
| ListenEvents.RESPONSE_TEXT_DELTA.value
|
||||
| "response.text.delta"
|
||||
):
|
||||
yield RealtimeTextEvent(
|
||||
service_type=event.type,
|
||||
service_event=event,
|
||||
text=StreamingTextContent(
|
||||
inner_content=event,
|
||||
text=event.delta, # type: ignore
|
||||
choice_index=0,
|
||||
),
|
||||
)
|
||||
case (
|
||||
ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE.value
|
||||
| "response.audio_transcript.done"
|
||||
| ListenEvents.RESPONSE_TEXT_DONE.value
|
||||
| "response.text.done"
|
||||
):
|
||||
# Don't yield RealtimeTextEvent here — the deltas already streamed all
|
||||
# the text. Emitting the full text again would cause duplicate output
|
||||
# for any consumer that prints every RealtimeTextEvent.
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED.value:
|
||||
if event.item.type == "function_call" and event.item.call_id and event.item.name: # type: ignore
|
||||
self._call_id_to_function_map[event.item.call_id] = event.item.name # type: ignore
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA.value:
|
||||
yield RealtimeFunctionCallEvent(
|
||||
service_type=event.type,
|
||||
service_event=event,
|
||||
function_call=FunctionCallContent(
|
||||
id=event.item_id, # type: ignore
|
||||
name=self._call_id_to_function_map[event.call_id], # type: ignore
|
||||
arguments=event.delta, # type: ignore
|
||||
index=event.output_index, # type: ignore
|
||||
metadata={"call_id": event.call_id}, # type: ignore
|
||||
inner_content=event,
|
||||
),
|
||||
)
|
||||
case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE.value:
|
||||
async for parsed_event in self._parse_function_call_arguments_done(event): # type: ignore
|
||||
if parsed_event:
|
||||
yield parsed_event
|
||||
case ListenEvents.ERROR.value:
|
||||
# In GA API, event.error is a dict instead of an object
|
||||
error_info = event.error if isinstance(event.error, dict) else event.error.model_dump() # type: ignore
|
||||
logger.error("Error received: %s", error_info) # type: ignore
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
case ListenEvents.SESSION_CREATED.value | ListenEvents.SESSION_UPDATED.value:
|
||||
logger.info("Session created or updated, session: %s", event.session.model_dump_json()) # type: ignore
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
case _:
|
||||
logger.debug(f"Received event: {event}")
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
|
||||
@override
|
||||
async def update_session(
|
||||
self,
|
||||
chat_history: ChatHistory | None = None,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
create_response: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Update the session in the service.
|
||||
|
||||
Args:
|
||||
chat_history: Chat history.
|
||||
settings: Prompt execution settings, if kernel is linked to the service or passed as
|
||||
Kwargs, it will be used to update the settings for function calling.
|
||||
create_response: Create a response, get the model to start responding, default is False.
|
||||
kwargs: Additional arguments, if 'kernel' or 'plugins' is passed, it will be used to update the
|
||||
settings for function calling, others will be ignored.
|
||||
|
||||
"""
|
||||
if kwargs:
|
||||
if self._create_kwargs:
|
||||
kwargs = {**self._create_kwargs, **kwargs}
|
||||
else:
|
||||
kwargs = self._create_kwargs or {}
|
||||
if settings:
|
||||
self._current_settings = settings
|
||||
if "kernel" in kwargs:
|
||||
self._kernel = kwargs["kernel"]
|
||||
if "plugins" in kwargs:
|
||||
self._add_plugin_to_kernel(kwargs["plugins"])
|
||||
|
||||
if self._current_settings:
|
||||
if self._kernel:
|
||||
self._current_settings = prepare_settings_for_function_calling(
|
||||
self._current_settings,
|
||||
self.get_prompt_execution_settings_class(),
|
||||
self._update_function_choice_settings_callback(),
|
||||
kernel=self._kernel,
|
||||
)
|
||||
await self.send(
|
||||
RealtimeEvent(
|
||||
service_type=SendEvents.SESSION_UPDATE,
|
||||
service_event={"settings": self._current_settings},
|
||||
)
|
||||
)
|
||||
|
||||
if chat_history and len(chat_history) > 0:
|
||||
for msg in chat_history.messages:
|
||||
for item in msg.items:
|
||||
match item:
|
||||
case TextContent():
|
||||
await self.send(
|
||||
RealtimeTextEvent(service_type=SendEvents.CONVERSATION_ITEM_CREATE, text=item)
|
||||
)
|
||||
case FunctionCallContent():
|
||||
await self.send(
|
||||
RealtimeFunctionCallEvent(
|
||||
service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_call=item
|
||||
)
|
||||
)
|
||||
case FunctionResultContent():
|
||||
await self.send(
|
||||
RealtimeFunctionResultEvent(
|
||||
service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_result=item
|
||||
)
|
||||
)
|
||||
case _:
|
||||
logger.error("Unsupported item type: %s", item)
|
||||
|
||||
if create_response or kwargs.get("create_response", False) is True:
|
||||
await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
|
||||
|
||||
def _add_plugin_to_kernel(self, plugins: list[object] | dict[str, object]) -> None:
|
||||
if not self._kernel:
|
||||
self._kernel = Kernel()
|
||||
if isinstance(plugins, list):
|
||||
plugins = {p.__class__.__name__: p for p in plugins}
|
||||
self._kernel.add_plugins(plugins)
|
||||
|
||||
async def _parse_function_call_arguments_done(
|
||||
self,
|
||||
event: ResponseFunctionCallArgumentsDoneEvent,
|
||||
) -> AsyncGenerator[RealtimeEvents | None]:
|
||||
"""Handle response function call done.
|
||||
|
||||
This always yields at least 1 event, either a RealtimeEvent or a RealtimeFunctionResultEvent with the raw event.
|
||||
|
||||
It then also yields any function results both back to the service, through `send` and to the developer.
|
||||
|
||||
"""
|
||||
# Step 1: check if function calling enabled:
|
||||
if not self._kernel or (
|
||||
self._current_settings
|
||||
and self._current_settings.function_choice_behavior
|
||||
and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions
|
||||
):
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
return
|
||||
# Step 2: check if there is a function that can be found.
|
||||
try:
|
||||
plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1)
|
||||
except ValueError:
|
||||
logger.error("Function call needs to have a plugin name and function name")
|
||||
yield RealtimeEvent(service_type=event.type, service_event=event)
|
||||
return
|
||||
|
||||
# Step 3: Parse into the function call content, and yield that.
|
||||
item = FunctionCallContent(
|
||||
id=event.item_id,
|
||||
plugin_name=plugin_name,
|
||||
function_name=function_name,
|
||||
arguments=event.arguments,
|
||||
index=event.output_index,
|
||||
metadata={"call_id": event.call_id},
|
||||
)
|
||||
yield RealtimeFunctionCallEvent(
|
||||
service_type=ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, function_call=item, service_event=event
|
||||
)
|
||||
|
||||
# Step 4: Invoke the function call
|
||||
# Fail closed: only invoke when a function choice behavior is available so the allowlist can be enforced.
|
||||
# Without it, kernel.invoke_function_call would skip allowlist validation and execute any named function.
|
||||
function_behavior = self._current_settings.function_choice_behavior if self._current_settings else None
|
||||
if function_behavior is None:
|
||||
logger.warning(
|
||||
"Skipping function call '%s-%s' because no function choice behavior is configured; "
|
||||
"allowlist validation cannot be enforced.",
|
||||
plugin_name,
|
||||
function_name,
|
||||
)
|
||||
created_output = FunctionResultContent.from_function_call_content_and_result(
|
||||
function_call_content=item,
|
||||
result="Function call was not invoked because function choice behavior is not configured.",
|
||||
)
|
||||
result = RealtimeFunctionResultEvent(
|
||||
service_type=SendEvents.CONVERSATION_ITEM_CREATE,
|
||||
function_result=created_output,
|
||||
)
|
||||
await self.send(result)
|
||||
await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
|
||||
yield result
|
||||
return
|
||||
chat_history = ChatHistory()
|
||||
await self._kernel.invoke_function_call(item, chat_history, function_behavior=function_behavior)
|
||||
created_output: FunctionResultContent = chat_history.messages[-1].items[0] # type: ignore
|
||||
# Step 5: Create the function result event
|
||||
result = RealtimeFunctionResultEvent(
|
||||
service_type=SendEvents.CONVERSATION_ITEM_CREATE,
|
||||
function_result=created_output,
|
||||
)
|
||||
# Step 6: send the result to the service and call `create response`
|
||||
await self.send(result)
|
||||
await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
|
||||
# Step 7: yield the function result back to the developer as well
|
||||
yield result
|
||||
|
||||
async def _send(self, event: RealtimeClientEvent) -> None:
|
||||
"""Send an event to the service."""
|
||||
raise NotImplementedError
|
||||
|
||||
@override
|
||||
async def send(self, event: RealtimeEvents, **kwargs: Any) -> None:
|
||||
match event:
|
||||
case RealtimeAudioEvent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, audio=event.audio.data_string
|
||||
)
|
||||
)
|
||||
case RealtimeTextEvent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=SendEvents.CONVERSATION_ITEM_CREATE,
|
||||
item=RealtimeConversationItemUserMessage(
|
||||
type="message",
|
||||
content=[
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": event.text.text,
|
||||
}
|
||||
],
|
||||
role="user",
|
||||
),
|
||||
)
|
||||
)
|
||||
case RealtimeFunctionCallEvent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=SendEvents.CONVERSATION_ITEM_CREATE,
|
||||
item=RealtimeConversationItemFunctionCall(
|
||||
type="function_call",
|
||||
name=event.function_call.name or event.function_call.function_name,
|
||||
arguments=""
|
||||
if not event.function_call.arguments
|
||||
else event.function_call.arguments
|
||||
if isinstance(event.function_call.arguments, str)
|
||||
else json.dumps(event.function_call.arguments),
|
||||
call_id=event.function_call.metadata.get("call_id"),
|
||||
),
|
||||
)
|
||||
)
|
||||
case RealtimeFunctionResultEvent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=SendEvents.CONVERSATION_ITEM_CREATE,
|
||||
item=RealtimeConversationItemFunctionCallOutput(
|
||||
type="function_call_output",
|
||||
output=event.function_result.result,
|
||||
call_id=event.function_result.metadata.get("call_id"),
|
||||
),
|
||||
)
|
||||
)
|
||||
case _:
|
||||
data = event.service_event
|
||||
match event.service_type:
|
||||
case SendEvents.SESSION_UPDATE:
|
||||
if not data:
|
||||
logger.error("Event data is empty")
|
||||
return
|
||||
settings = data.get("settings", None)
|
||||
if not settings:
|
||||
logger.error("Event data does not contain 'settings'")
|
||||
return
|
||||
try:
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to properly create settings from passed settings: {settings}, error: {e}"
|
||||
)
|
||||
return
|
||||
assert isinstance(settings, self.get_prompt_execution_settings_class()) # nosec
|
||||
if not settings.ai_model_id: # type: ignore
|
||||
settings.ai_model_id = self.ai_model_id # type: ignore
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
session=settings.prepare_settings_dict(),
|
||||
)
|
||||
)
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_APPEND:
|
||||
if not data or "audio" not in data:
|
||||
logger.error("Event data does not contain 'audio'")
|
||||
return
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
audio=data["audio"],
|
||||
)
|
||||
)
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_COMMIT:
|
||||
await self._send(_create_openai_realtime_client_event(event_type=event.service_type))
|
||||
case SendEvents.INPUT_AUDIO_BUFFER_CLEAR:
|
||||
await self._send(_create_openai_realtime_client_event(event_type=event.service_type))
|
||||
case SendEvents.CONVERSATION_ITEM_CREATE:
|
||||
if not data or "item" not in data:
|
||||
logger.error("Event data does not contain 'item'")
|
||||
return
|
||||
content = data["item"]
|
||||
contents = content.items if isinstance(content, ChatMessageContent) else [content]
|
||||
for item in contents:
|
||||
match item:
|
||||
case TextContent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
item=RealtimeConversationItemUserMessage(
|
||||
type="message",
|
||||
content=[
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": item.text,
|
||||
}
|
||||
],
|
||||
role="user",
|
||||
),
|
||||
)
|
||||
)
|
||||
case FunctionCallContent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
item=RealtimeConversationItemFunctionCall(
|
||||
type="function_call",
|
||||
name=item.name or item.function_name,
|
||||
arguments=""
|
||||
if not item.arguments
|
||||
else item.arguments
|
||||
if isinstance(item.arguments, str)
|
||||
else json.dumps(item.arguments),
|
||||
call_id=item.metadata.get("call_id"),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
case FunctionResultContent():
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
item=RealtimeConversationItemFunctionCallOutput(
|
||||
type="function_call_output",
|
||||
output=item.result,
|
||||
call_id=item.metadata.get("call_id"),
|
||||
),
|
||||
)
|
||||
)
|
||||
case SendEvents.CONVERSATION_ITEM_TRUNCATE:
|
||||
if not data or "item_id" not in data:
|
||||
logger.error("Event data does not contain 'item_id'")
|
||||
return
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
item_id=data["item_id"],
|
||||
content_index=0,
|
||||
audio_end_ms=data.get("audio_end_ms", 0),
|
||||
)
|
||||
)
|
||||
case SendEvents.CONVERSATION_ITEM_DELETE:
|
||||
if not data or "item_id" not in data:
|
||||
logger.error("Event data does not contain 'item_id'")
|
||||
return
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
item_id=data["item_id"],
|
||||
)
|
||||
)
|
||||
case SendEvents.RESPONSE_CREATE:
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type, event_id=data.get("event_id", None) if data else None
|
||||
)
|
||||
)
|
||||
case SendEvents.RESPONSE_CANCEL:
|
||||
await self._send(
|
||||
_create_openai_realtime_client_event(
|
||||
event_type=event.service_type,
|
||||
response_id=data.get("response_id", None) if data else None,
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
return OpenAIRealtimeExecutionSettings
|
||||
|
||||
@override
|
||||
def _update_function_choice_settings_callback(
|
||||
self,
|
||||
) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]:
|
||||
return update_settings_from_function_call_configuration
|
||||
|
||||
|
||||
# region WebRTC
|
||||
@experimental
|
||||
class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase):
|
||||
"""OpenAI WebRTC Realtime service."""
|
||||
|
||||
peer_connection: RTCPeerConnection | None = None
|
||||
data_channel: RTCDataChannel | None = None
|
||||
audio_track: MediaStreamTrack | None = None
|
||||
_receive_buffer: asyncio.Queue[RealtimeEvents] = PrivateAttr(default_factory=asyncio.Queue)
|
||||
|
||||
@override
|
||||
async def receive(
|
||||
self,
|
||||
audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> AsyncGenerator[RealtimeEvents, None]:
|
||||
if audio_output_callback:
|
||||
self.audio_output_callback = audio_output_callback
|
||||
while True:
|
||||
yield await self._receive_buffer.get()
|
||||
|
||||
async def _send(self, event: RealtimeClientEvent) -> None:
|
||||
if not self.data_channel:
|
||||
logger.error("Data channel not initialized")
|
||||
return
|
||||
while self.data_channel.readyState != "open":
|
||||
await asyncio.sleep(0.1)
|
||||
try:
|
||||
# Handle session update specially to exclude type field for WebRTC
|
||||
if hasattr(event, "type") and event.type == "session.update":
|
||||
event_dict = event.model_dump(exclude_none=True)
|
||||
# Remove fields that aren't allowed in session updates for WebRTC compatibility
|
||||
# Audio configuration should be set during session creation, not updates
|
||||
session_dict = event_dict.get("session")
|
||||
if session_dict and isinstance(session_dict, dict):
|
||||
# Only keep fields that are allowed in session updates
|
||||
# Note: output_modalities is not allowed in WebRTC session updates
|
||||
allowed_fields = {
|
||||
"type",
|
||||
"instructions",
|
||||
"model",
|
||||
"max_output_tokens",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"prompt",
|
||||
"tracing",
|
||||
"truncation",
|
||||
}
|
||||
event_dict["session"] = {k: v for k, v in session_dict.items() if k in allowed_fields}
|
||||
|
||||
self.data_channel.send(json.dumps(event_dict))
|
||||
else:
|
||||
self.data_channel.send(event.model_dump_json(exclude_none=True))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send event {event} with error: {e!s}")
|
||||
|
||||
@override
|
||||
async def create_session(
|
||||
self,
|
||||
chat_history: "ChatHistory | None" = None,
|
||||
settings: "PromptExecutionSettings | None" = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Create a session in the service."""
|
||||
if not self.audio_track:
|
||||
raise Exception("Audio track not initialized")
|
||||
self.peer_connection = RTCPeerConnection(
|
||||
configuration=RTCConfiguration(iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")])
|
||||
)
|
||||
|
||||
# track is the audio track being returned from the service
|
||||
self.peer_connection.add_listener("track", self._on_track)
|
||||
|
||||
# data channel is used to send and receive messages
|
||||
self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json")
|
||||
self.data_channel.add_listener("message", self._on_data)
|
||||
|
||||
# this is the incoming audio, which sends audio to the service
|
||||
self.peer_connection.addTransceiver(self.audio_track)
|
||||
|
||||
offer = await self.peer_connection.createOffer()
|
||||
await self.peer_connection.setLocalDescription(offer)
|
||||
|
||||
try:
|
||||
ephemeral_token = await self._get_ephemeral_token()
|
||||
headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"}
|
||||
headers = prepend_semantic_kernel_to_user_agent(headers)
|
||||
|
||||
async with (
|
||||
ClientSession() as session,
|
||||
session.post(self._get_webrtc_url(), headers=headers, data=offer.sdp) as response,
|
||||
):
|
||||
if response.status not in [200, 201]:
|
||||
error_text = await response.text()
|
||||
raise Exception(f"OpenAI WebRTC error: {error_text}")
|
||||
|
||||
sdp_answer = await response.text()
|
||||
answer = RTCSessionDescription(sdp=sdp_answer, type="answer")
|
||||
await self.peer_connection.setRemoteDescription(answer)
|
||||
logger.info("Connected to OpenAI WebRTC")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to OpenAI: {e!s}")
|
||||
raise
|
||||
|
||||
await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
|
||||
|
||||
@override
|
||||
async def close_session(self) -> None:
|
||||
"""Close the session in the service."""
|
||||
if self.peer_connection:
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await self.peer_connection.close()
|
||||
self.peer_connection = None
|
||||
if self.data_channel:
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
self.data_channel.close()
|
||||
self.data_channel = None
|
||||
|
||||
async def _on_track(self, track: "MediaStreamTrack") -> None:
|
||||
logger.debug(f"Received {track.kind} track from remote")
|
||||
if track.kind != "audio":
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
# This is a MediaStreamTrack, so the type is AudioFrame
|
||||
# this might need to be updated if video becomes part of this
|
||||
frame: AudioFrame = await track.recv() # type: ignore
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting audio frame: {e!s}")
|
||||
break
|
||||
|
||||
try:
|
||||
if self.audio_output_callback:
|
||||
await self.audio_output_callback(frame.to_ndarray())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error playing remote audio frame: {e!s}")
|
||||
try:
|
||||
await self._receive_buffer.put(
|
||||
RealtimeAudioEvent(
|
||||
audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame),
|
||||
service_event=frame,
|
||||
service_type=ListenEvents.RESPONSE_AUDIO_DELTA,
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing remote audio frame: {e!s}")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def _on_data(self, data: str) -> None:
|
||||
"""This method is called whenever a data channel message is received.
|
||||
|
||||
The data is parsed into a RealtimeServerEvent (by OpenAI code) and then processed.
|
||||
Audio data is not send through this channel, use _on_track for that.
|
||||
"""
|
||||
try:
|
||||
event = cast(
|
||||
RealtimeServerEvent,
|
||||
construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse event {data} with error: {e!s}")
|
||||
return
|
||||
async for parsed_event in self._parse_event(event):
|
||||
await self._receive_buffer.put(parsed_event)
|
||||
|
||||
async def _get_ephemeral_token(self) -> str:
|
||||
"""Get an ephemeral token from OpenAI.
|
||||
|
||||
GA endpoint: POST /v1/realtime/client_secrets
|
||||
Request body: {"session": {"type": "realtime", "model": "<model>"}}
|
||||
Response: {"value": "<token>", "expires_at": ..., "session": {...}}
|
||||
"""
|
||||
data = {
|
||||
"session": {
|
||||
"type": "realtime",
|
||||
"model": self.ai_model_id,
|
||||
}
|
||||
}
|
||||
headers, url = self._get_ephemeral_token_headers_and_url()
|
||||
headers = prepend_semantic_kernel_to_user_agent(headers)
|
||||
try:
|
||||
async with (
|
||||
ClientSession() as session,
|
||||
session.post(url, headers=headers, json=data) as response,
|
||||
):
|
||||
if response.status not in [200, 201]:
|
||||
error_text = await response.text()
|
||||
raise Exception(f"Failed to get ephemeral token: {error_text}")
|
||||
|
||||
result = await response.json()
|
||||
return result["value"]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get ephemeral token: {e!s}")
|
||||
raise
|
||||
|
||||
def _get_ephemeral_token_headers_and_url(self) -> tuple[dict[str, str], str]:
|
||||
"""Get the headers and URL for the ephemeral token."""
|
||||
return {
|
||||
"Authorization": f"Bearer {self.client.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}, f"{self.client.realtime._client.base_url}/realtime/client_secrets"
|
||||
|
||||
def _get_webrtc_url(self) -> str:
|
||||
"""Get the WebRTC URL.
|
||||
|
||||
GA endpoint: POST /v1/realtime/calls?model=<model>
|
||||
"""
|
||||
return f"{self.client.realtime._client.base_url}/realtime/calls?model={self.ai_model_id}"
|
||||
|
||||
|
||||
# region Websocket
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAIRealtimeWebsocketBase(OpenAIRealtimeBase):
|
||||
"""OpenAI Realtime service."""
|
||||
|
||||
protocol: ClassVar[Literal["websocket"]] = "websocket" # type: ignore
|
||||
connection: AsyncRealtimeConnection | None = None
|
||||
connected: asyncio.Event = Field(default_factory=asyncio.Event)
|
||||
|
||||
@override
|
||||
async def receive(
|
||||
self,
|
||||
audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> AsyncGenerator[RealtimeEvents, None]:
|
||||
if audio_output_callback:
|
||||
self.audio_output_callback = audio_output_callback
|
||||
await self.connected.wait()
|
||||
if not self.connection:
|
||||
raise ValueError("Connection is not established.")
|
||||
|
||||
async for event in self.connection:
|
||||
if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value:
|
||||
if self.audio_output_callback:
|
||||
await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16))
|
||||
yield RealtimeAudioEvent(
|
||||
audio=AudioContent(data=event.delta, data_format="base64", inner_content=event),
|
||||
service_type=event.type,
|
||||
service_event=event,
|
||||
)
|
||||
continue
|
||||
async for realtime_event in self._parse_event(event):
|
||||
yield realtime_event
|
||||
|
||||
async def _send(self, event: RealtimeClientEvent) -> None:
|
||||
await self.connected.wait()
|
||||
if not self.connection:
|
||||
raise ValueError("Connection is not established.")
|
||||
try:
|
||||
await self.connection.send(event)
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending response: {e!s}")
|
||||
|
||||
@override
|
||||
async def create_session(
|
||||
self,
|
||||
chat_history: "ChatHistory | None" = None,
|
||||
settings: "PromptExecutionSettings | None" = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Create a session in the service."""
|
||||
self.connection = await self.client.realtime.connect(
|
||||
model=self.ai_model_id, extra_headers={USER_AGENT: SEMANTIC_KERNEL_USER_AGENT}
|
||||
).enter()
|
||||
self.connected.set()
|
||||
await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
|
||||
|
||||
@override
|
||||
async def close_session(self) -> None:
|
||||
"""Close the session in the service."""
|
||||
if self.connected.is_set() and self.connection:
|
||||
await self.connection.close()
|
||||
self.connection = None
|
||||
self.connected.clear()
|
||||
@@ -0,0 +1,117 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text_base import OpenAIAudioToTextBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="AzureAudioToText")
|
||||
|
||||
|
||||
class AzureAudioToText(AzureOpenAIConfigBase, OpenAIAudioToTextBase):
|
||||
"""Azure audio to text service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureAudioToText service.
|
||||
|
||||
Args:
|
||||
service_id: The service ID. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(audio_to_text_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure AD token for authentication. (Optional)
|
||||
ad_token_provider: Azure AD Token provider. (Optional)
|
||||
token_endpoint: The Azure AD token endpoint. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
api_key=api_key,
|
||||
audio_to_text_deployment_name=deployment_name,
|
||||
endpoint=endpoint,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
|
||||
if not azure_openai_settings.audio_to_text_deployment_name:
|
||||
raise ServiceInitializationError("The Azure OpenAI audio to text deployment name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.audio_to_text_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.AUDIO_TO_TEXT,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: deployment_name, endpoint, api_key
|
||||
and optionally: api_version, ad_auth
|
||||
"""
|
||||
return cls(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,221 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from typing import Any, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import (
|
||||
AzureChatPromptExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base import OpenAIChatCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.function_call_content import FunctionCallContent
|
||||
from semantic_kernel.contents.function_result_content import FunctionResultContent
|
||||
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.contents.utils.finish_reason import FinishReason
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
TChatMessageContent = TypeVar("TChatMessageContent", ChatMessageContent, StreamingChatMessageContent)
|
||||
|
||||
|
||||
class AzureChatCompletion(AzureOpenAIConfigBase, OpenAIChatCompletionBase, OpenAITextCompletionBase):
|
||||
"""Azure Chat completion class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
instruction_role: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureChatCompletion service.
|
||||
|
||||
Args:
|
||||
service_id (str | None): The service ID for the Azure deployment. (Optional)
|
||||
api_key (str | None): The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name (str | None): The optional deployment. If provided, will override the value
|
||||
(chat_deployment_name) in the env vars or .env file.
|
||||
endpoint (str | None): The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url (str | None): The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version (str | None): The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token (str | None): The Azure Active Directory token. (Optional)
|
||||
ad_token_provider (AsyncAzureADTokenProvider): The Azure Active Directory token provider. (Optional)
|
||||
token_endpoint (str | None): The token endpoint to request an Azure token. (Optional)
|
||||
default_headers (Mapping[str, str]): The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (AsyncAzureOpenAI | None): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to using env vars.
|
||||
env_file_encoding (str | None): The encoding of the environment settings file, defaults to 'utf-8'.
|
||||
instruction_role (str | None): The role to use for 'instruction' messages, for example, summarization
|
||||
prompts could use `developer` or `system`. (Optional)
|
||||
credential (TokenCredential): The credential to use for authentication.
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
endpoint=endpoint,
|
||||
chat_deployment_name=deployment_name,
|
||||
api_version=api_version,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise ServiceInitializationError(f"Failed to validate settings: {exc}") from exc
|
||||
|
||||
if not azure_openai_settings.chat_deployment_name:
|
||||
raise ServiceInitializationError("chat_deployment_name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.chat_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.CHAT,
|
||||
client=async_client,
|
||||
instruction_role=instruction_role,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, settings: dict[str, Any]) -> "AzureChatCompletion":
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: service_id, and optionally:
|
||||
ad_auth, ad_token_provider, default_headers
|
||||
"""
|
||||
return AzureChatCompletion(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
"""Create a request settings object."""
|
||||
return AzureChatPromptExecutionSettings
|
||||
|
||||
def _create_chat_message_content(
|
||||
self, response: ChatCompletion, choice: Choice, response_metadata: dict[str, Any]
|
||||
) -> ChatMessageContent:
|
||||
"""Create an Azure chat message content object from a choice."""
|
||||
content = super()._create_chat_message_content(response, choice, response_metadata)
|
||||
return self._add_tool_message_to_chat_message_content(content, choice)
|
||||
|
||||
def _create_streaming_chat_message_content(
|
||||
self,
|
||||
chunk: ChatCompletionChunk,
|
||||
choice: ChunkChoice,
|
||||
chunk_metadata: dict[str, Any],
|
||||
function_invoke_attempt: int = 0,
|
||||
) -> "StreamingChatMessageContent":
|
||||
"""Create an Azure streaming chat message content object from a choice."""
|
||||
content = super()._create_streaming_chat_message_content(chunk, choice, chunk_metadata, function_invoke_attempt)
|
||||
assert isinstance(content, StreamingChatMessageContent) and isinstance(choice, ChunkChoice) # nosec
|
||||
return self._add_tool_message_to_chat_message_content(content, choice)
|
||||
|
||||
def _add_tool_message_to_chat_message_content(
|
||||
self,
|
||||
content: TChatMessageContent,
|
||||
choice: Choice | ChunkChoice,
|
||||
) -> TChatMessageContent:
|
||||
if tool_message := self._get_tool_message_from_chat_choice(choice=choice):
|
||||
if not isinstance(tool_message, dict):
|
||||
# try to json, to ensure it is a dictionary
|
||||
try:
|
||||
tool_message = json.loads(tool_message)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Tool message is not a dictionary, ignore context.")
|
||||
return content
|
||||
function_call = FunctionCallContent(
|
||||
id=str(uuid4()),
|
||||
name="Azure-OnYourData",
|
||||
arguments=json.dumps({"query": tool_message.get("intent", [])}),
|
||||
)
|
||||
result = FunctionResultContent.from_function_call_content_and_result(
|
||||
result=tool_message["citations"], function_call_content=function_call
|
||||
)
|
||||
content.items.insert(0, function_call)
|
||||
content.items.insert(1, result)
|
||||
return content
|
||||
|
||||
def _get_tool_message_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[str, Any] | None:
|
||||
"""Get the tool message from a choice."""
|
||||
content = choice.message if isinstance(choice, Choice) else choice.delta
|
||||
# When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
|
||||
if content and content.model_extra is not None:
|
||||
return content.model_extra.get("context", None)
|
||||
# openai allows extra content, so model_extra will be a dict, but we need to check anyway, but no way to test.
|
||||
return None # pragma: no cover
|
||||
|
||||
@staticmethod
|
||||
def split_message(message: "ChatMessageContent") -> list["ChatMessageContent"]:
|
||||
"""Split an Azure On Your Data response into separate ChatMessageContents.
|
||||
|
||||
If the message does not have three contents, and those three are one each of:
|
||||
FunctionCallContent, FunctionResultContent, and TextContent,
|
||||
it will not return three messages, potentially only one or two.
|
||||
|
||||
The order of the returned messages is as expected by OpenAI.
|
||||
"""
|
||||
if len(message.items) != 3:
|
||||
return [message]
|
||||
messages = {"tool_call": deepcopy(message), "tool_result": deepcopy(message), "assistant": deepcopy(message)}
|
||||
for key, msg in messages.items():
|
||||
if key == "tool_call":
|
||||
msg.items = [item for item in msg.items if isinstance(item, FunctionCallContent)]
|
||||
msg.finish_reason = FinishReason.FUNCTION_CALL
|
||||
if key == "tool_result":
|
||||
msg.items = [item for item in msg.items if isinstance(item, FunctionResultContent)]
|
||||
if key == "assistant":
|
||||
msg.items = [item for item in msg.items if isinstance(item, TextContent)]
|
||||
return [messages["tool_call"], messages["tool_result"], messages["assistant"]]
|
||||
@@ -0,0 +1,151 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from copy import copy
|
||||
from typing import Any
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from pydantic import ConfigDict, validate_call
|
||||
from pydantic_core import Url
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.const import DEFAULT_AZURE_API_VERSION
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler, OpenAIModelTypes
|
||||
from semantic_kernel.const import USER_AGENT
|
||||
from semantic_kernel.exceptions import ServiceInitializationError
|
||||
from semantic_kernel.kernel_pydantic import HttpsUrl
|
||||
from semantic_kernel.utils.authentication.entra_id_authentication import get_entra_auth_token
|
||||
from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AzureOpenAIConfigBase(OpenAIHandler):
|
||||
"""Internal class for configuring a connection to an Azure OpenAI service."""
|
||||
|
||||
@validate_call(config=ConfigDict(arbitrary_types_allowed=True))
|
||||
def __init__(
|
||||
self,
|
||||
deployment_name: str,
|
||||
ai_model_type: OpenAIModelTypes,
|
||||
endpoint: HttpsUrl | None = None,
|
||||
base_url: Url | None = None,
|
||||
api_version: str = DEFAULT_AZURE_API_VERSION,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: Callable[[], str | Awaitable[str]] | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
client: AsyncAzureOpenAI | None = None,
|
||||
instruction_role: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Internal class for configuring a connection to an Azure OpenAI service.
|
||||
|
||||
The `validate_call` decorator is used with a configuration that allows arbitrary types.
|
||||
This is necessary for types like `HttpsUrl` and `OpenAIModelTypes`.
|
||||
|
||||
Args:
|
||||
deployment_name (str): Name of the deployment.
|
||||
ai_model_type (OpenAIModelTypes): The type of OpenAI model to deploy.
|
||||
endpoint (HttpsUrl): The specific endpoint URL for the deployment. (Optional)
|
||||
base_url (Url): The base URL for Azure services. (Optional)
|
||||
api_version (str): Azure API version. Defaults to the defined DEFAULT_AZURE_API_VERSION.
|
||||
service_id (str): Service ID for the deployment. (Optional)
|
||||
api_key (str): API key for Azure services. (Optional)
|
||||
ad_token (str): Azure AD token for authentication. (Optional)
|
||||
ad_token_provider (Callable[[], Union[str, Awaitable[str]]]): A callable
|
||||
or coroutine function providing Azure AD tokens. (Optional)
|
||||
token_endpoint (str): Azure AD token endpoint use to get the token. (Optional)
|
||||
default_headers (Union[Mapping[str, str], None]): Default headers for HTTP requests. (Optional)
|
||||
client (AsyncAzureOpenAI): An existing client to use. (Optional)
|
||||
instruction_role (str | None): The role to use for 'instruction' messages, for example, summarization
|
||||
prompts could use `developer` or `system`. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
kwargs: Additional keyword arguments.
|
||||
|
||||
"""
|
||||
# Merge APP_INFO into the headers if it exists
|
||||
merged_headers = dict(copy(default_headers)) if default_headers else {}
|
||||
if APP_INFO:
|
||||
merged_headers.update(APP_INFO)
|
||||
merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers)
|
||||
|
||||
if not client:
|
||||
# If the client is None, the api_key is none, the ad_token is none, and the ad_token_provider is none,
|
||||
# then we will attempt to get the ad_token using the default endpoint specified in the Azure OpenAI
|
||||
# settings.
|
||||
if not api_key and not ad_token_provider and not ad_token and token_endpoint and credential:
|
||||
ad_token = get_entra_auth_token(credential, token_endpoint)
|
||||
|
||||
if not api_key and not ad_token and not ad_token_provider and not credential:
|
||||
raise ServiceInitializationError(
|
||||
"Please provide either api_key, ad_token, ad_token_provider, credential or a client."
|
||||
)
|
||||
|
||||
if not endpoint and not base_url:
|
||||
raise ServiceInitializationError("Please provide an endpoint or a base_url")
|
||||
|
||||
args: dict[str, Any] = {
|
||||
"default_headers": merged_headers,
|
||||
}
|
||||
if api_version:
|
||||
args["api_version"] = api_version
|
||||
if ad_token:
|
||||
args["azure_ad_token"] = ad_token
|
||||
if ad_token_provider:
|
||||
args["azure_ad_token_provider"] = ad_token_provider
|
||||
if api_key:
|
||||
args["api_key"] = api_key
|
||||
if base_url:
|
||||
args["base_url"] = str(base_url)
|
||||
if endpoint and not base_url:
|
||||
args["azure_endpoint"] = str(endpoint)
|
||||
# TODO (eavanvalkenburg): Remove the check on model type when the package fixes: https://github.com/openai/openai-python/issues/2120
|
||||
if deployment_name and ai_model_type != OpenAIModelTypes.REALTIME:
|
||||
args["azure_deployment"] = deployment_name
|
||||
|
||||
if "websocket_base_url" in kwargs:
|
||||
args["websocket_base_url"] = kwargs.pop("websocket_base_url")
|
||||
|
||||
client = AsyncAzureOpenAI(**args)
|
||||
args = {
|
||||
"ai_model_id": deployment_name,
|
||||
"client": client,
|
||||
"ai_model_type": ai_model_type,
|
||||
}
|
||||
if service_id:
|
||||
args["service_id"] = service_id
|
||||
if instruction_role:
|
||||
args["instruction_role"] = instruction_role
|
||||
super().__init__(**args, **kwargs)
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
"""Convert the configuration to a dictionary."""
|
||||
client_settings = {
|
||||
"base_url": str(self.client.base_url),
|
||||
"api_version": self.client._custom_query["api-version"],
|
||||
"api_key": self.client.api_key,
|
||||
"ad_token": getattr(self.client, "_azure_ad_token", None),
|
||||
"ad_token_provider": getattr(self.client, "_azure_ad_token_provider", None),
|
||||
"default_headers": {k: v for k, v in self.client.default_headers.items() if k != USER_AGENT},
|
||||
}
|
||||
base = self.model_dump(
|
||||
exclude={
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"api_type",
|
||||
"org_id",
|
||||
"ai_model_type",
|
||||
"service_id",
|
||||
"client",
|
||||
},
|
||||
by_alias=True,
|
||||
exclude_none=True,
|
||||
)
|
||||
base.update(client_settings)
|
||||
return base
|
||||
@@ -0,0 +1,387 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import warnings
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from aiohttp import ClientSession
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from openai.resources.realtime.realtime import AsyncRealtimeConnection
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
|
||||
AzureRealtimeExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import (
|
||||
OpenAIRealtimeWebRTCBase,
|
||||
OpenAIRealtimeWebsocketBase,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.const import USER_AGENT
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT, prepend_semantic_kernel_to_user_agent
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiortc.mediastreams import MediaStreamTrack
|
||||
from numpy import ndarray
|
||||
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
logger = logging.getLogger("semantic_kernel.connectors.ai.open_ai.realtime")
|
||||
|
||||
|
||||
@experimental
|
||||
class AzureRealtimeWebsocket(OpenAIRealtimeWebsocketBase, AzureOpenAIConfigBase):
|
||||
"""Azure OpenAI Realtime service using WebSocket protocol."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
websocket_base_url: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize an AzureRealtimeWebsocket service.
|
||||
|
||||
Args:
|
||||
audio_output_callback: The audio output callback, optional.
|
||||
This should be a coroutine, that takes a ndarray with audio as input.
|
||||
The goal of this function is to allow you to play the audio with the
|
||||
least amount of latency possible, because it is called first before further processing.
|
||||
It can also be set in the `receive` method.
|
||||
Even when passed, the audio content will still be
|
||||
added to the receiving queue.
|
||||
service_id: The service ID for the Azure deployment. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(chat_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure Active Directory token. (Optional)
|
||||
ad_token_provider: The Azure Active Directory token provider. (Optional)
|
||||
token_endpoint: The token endpoint to request an Azure token. (Optional)
|
||||
websocket_base_url: The base URL for the WebSocket connection. (Optional)
|
||||
If not provided, the default URL will be used.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
kwargs: Additional arguments.
|
||||
This can include:
|
||||
kernel (Kernel): the kernel to use for function calls
|
||||
plugins (list[object] or dict[str, object]): the plugins to use for function calls
|
||||
settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
|
||||
chat_history (ChatHistory): the chat history to use for the session
|
||||
Otherwise they can also be passed to the context manager.
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
endpoint=endpoint,
|
||||
realtime_deployment_name=deployment_name,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not azure_openai_settings.realtime_deployment_name:
|
||||
raise ServiceInitializationError("The OpenAI realtime model ID is required.")
|
||||
super().__init__(
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
audio_output_callback=audio_output_callback,
|
||||
deployment_name=azure_openai_settings.realtime_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
ai_model_type=OpenAIModelTypes.REALTIME,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
websocket_base_url=websocket_base_url,
|
||||
credential=credential,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
|
||||
return AzureRealtimeExecutionSettings
|
||||
|
||||
@override
|
||||
async def create_session(
|
||||
self,
|
||||
chat_history: "ChatHistory | None" = None,
|
||||
settings: "PromptExecutionSettings | None" = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Create a session in the service.
|
||||
|
||||
The Azure GA Realtime endpoint (/openai/v1/realtime) does not accept
|
||||
the api-version query parameter. The openai SDK always adds it, so we
|
||||
bypass the SDK's _configure_realtime and build the connection directly.
|
||||
"""
|
||||
from websockets.asyncio.client import connect as ws_connect
|
||||
|
||||
# Build the GA WebSocket URL: wss://<resource>.openai.azure.com/openai/v1/realtime?model=<deployment-name>
|
||||
# Note: GA uses ?model= (not ?deployment= which was preview)
|
||||
# See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-websockets
|
||||
endpoint = str(self.client._base_url).rstrip("/") # type: ignore[attr-defined]
|
||||
if "/openai" in endpoint:
|
||||
endpoint = endpoint[: endpoint.index("/openai")]
|
||||
url = f"wss://{endpoint.split('://')[-1]}/openai/v1/realtime?model={self.ai_model_id}"
|
||||
|
||||
# Build auth headers
|
||||
auth_headers: dict[str, str] = {}
|
||||
if self.client.api_key and self.client.api_key != "<missing API key>":
|
||||
auth_headers["api-key"] = self.client.api_key
|
||||
else:
|
||||
token = await self.client._get_azure_ad_token() # type: ignore[attr-defined]
|
||||
if token:
|
||||
auth_headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
ws = await ws_connect(
|
||||
url,
|
||||
additional_headers={
|
||||
**auth_headers,
|
||||
USER_AGENT: SEMANTIC_KERNEL_USER_AGENT,
|
||||
},
|
||||
)
|
||||
|
||||
self.connection = AsyncRealtimeConnection(ws)
|
||||
self.connected.set()
|
||||
await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
|
||||
|
||||
|
||||
@experimental
|
||||
class AzureRealtimeWebRTC(OpenAIRealtimeWebRTCBase, AzureOpenAIConfigBase):
|
||||
"""Azure OpenAI Realtime service using WebRTC protocol."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_track: "MediaStreamTrack",
|
||||
region: str | None = None,
|
||||
audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize an AzureRealtimeWebRTC service.
|
||||
|
||||
Args:
|
||||
audio_track: The audio track to use for the service, only used by WebRTC.
|
||||
It can be any class that implements the AudioStreamTrack interface.
|
||||
region: Deprecated. No longer needed for GA Realtime API.
|
||||
Previously required for the preview WebRTC endpoint.
|
||||
audio_output_callback: The audio output callback, optional.
|
||||
This should be a coroutine, that takes a ndarray with audio as input.
|
||||
The goal of this function is to allow you to play the audio with the
|
||||
least amount of latency possible, because it is called first before further processing.
|
||||
It can also be set in the `receive` method.
|
||||
Even when passed, the audio content will still be
|
||||
added to the receiving queue.
|
||||
service_id: The service ID for the Azure deployment. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(chat_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure Active Directory token. (Optional)
|
||||
ad_token_provider: The Azure Active Directory token provider. (Optional)
|
||||
token_endpoint: The token endpoint to request an Azure token. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
kwargs: Additional arguments.
|
||||
This can include:
|
||||
kernel (Kernel): the kernel to use for function calls
|
||||
plugins (list[object] or dict[str, object]): the plugins to use for function calls
|
||||
settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
|
||||
chat_history (ChatHistory): the chat history to use for the session
|
||||
Otherwise they can also be passed to the context manager.
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
endpoint=endpoint,
|
||||
realtime_deployment_name=deployment_name,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not azure_openai_settings.realtime_deployment_name:
|
||||
raise ServiceInitializationError("The OpenAI realtime model ID is required.")
|
||||
if region is not None:
|
||||
warnings.warn(
|
||||
"The 'region' parameter is deprecated and no longer needed for the GA Realtime API. "
|
||||
"The WebRTC endpoint is now derived from the resource endpoint.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if audio_track:
|
||||
kwargs["audio_track"] = audio_track
|
||||
super().__init__(
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
audio_output_callback=audio_output_callback,
|
||||
deployment_name=azure_openai_settings.realtime_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
ai_model_type=OpenAIModelTypes.REALTIME,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
|
||||
return AzureRealtimeExecutionSettings
|
||||
|
||||
@override
|
||||
def _get_ephemeral_token_headers_and_url(self) -> tuple[dict[str, str], str]:
|
||||
"""Get the headers and URL for the ephemeral token.
|
||||
|
||||
Uses the GA endpoint format: POST /openai/v1/realtime/client_secrets
|
||||
See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
|
||||
"""
|
||||
endpoint = str(self.client._base_url).rstrip("/") # type: ignore[attr-defined]
|
||||
# Strip any trailing path segments to get the base Azure resource URL
|
||||
# base_url typically looks like https://<resource>.openai.azure.com/openai/...
|
||||
# We need: https://<resource>.openai.azure.com/openai/v1/realtime/client_secrets
|
||||
if "/openai" in endpoint:
|
||||
endpoint = endpoint[: endpoint.index("/openai")]
|
||||
url = f"{endpoint}/openai/v1/realtime/client_secrets"
|
||||
|
||||
if self.client.api_key and self.client.api_key != "<missing API key>":
|
||||
return (
|
||||
{
|
||||
"api-key": self.client.api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
url,
|
||||
)
|
||||
if self.client._azure_ad_token is not None: # type: ignore[attr-defined]
|
||||
return (
|
||||
{
|
||||
"Authorization": f"Bearer {self.client._azure_ad_token}", # type: ignore[attr-defined]
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
url,
|
||||
)
|
||||
raise ServiceInitializationError("No API key or Azure AD token available for ephemeral token request.")
|
||||
|
||||
@override
|
||||
async def _get_ephemeral_token(self) -> str:
|
||||
"""Get an ephemeral token from Azure OpenAI.
|
||||
|
||||
Azure GA requires a nested session object:
|
||||
{"session": {"type": "realtime", "model": "<deployment>"}}
|
||||
And returns the token directly as {"value": "..."} rather than
|
||||
OpenAI's {"client_secret": {"value": "..."}}.
|
||||
See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
|
||||
"""
|
||||
data = {
|
||||
"session": {
|
||||
"type": "realtime",
|
||||
"model": self.ai_model_id,
|
||||
}
|
||||
}
|
||||
headers, url = self._get_ephemeral_token_headers_and_url()
|
||||
headers = prepend_semantic_kernel_to_user_agent(headers)
|
||||
try:
|
||||
async with (
|
||||
ClientSession() as session,
|
||||
session.post(url, headers=headers, json=data) as response,
|
||||
):
|
||||
if response.status not in [200, 201]:
|
||||
error_text = await response.text()
|
||||
raise Exception(f"Failed to get ephemeral token: {error_text}")
|
||||
|
||||
result = await response.json()
|
||||
# Azure GA format returns {"value": "token"} directly
|
||||
return result["value"]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get ephemeral token: {e!s}")
|
||||
raise
|
||||
|
||||
@override
|
||||
def _get_webrtc_url(self) -> str:
|
||||
"""Get the WebRTC URL.
|
||||
|
||||
Uses the GA endpoint format: /openai/v1/realtime/calls
|
||||
See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
|
||||
"""
|
||||
endpoint = str(self.client._base_url).rstrip("/") # type: ignore[attr-defined]
|
||||
if "/openai" in endpoint:
|
||||
endpoint = endpoint[: endpoint.index("/openai")]
|
||||
return f"{endpoint}/openai/v1/realtime/calls"
|
||||
@@ -0,0 +1,127 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
if sys.version_info >= (3, 13):
|
||||
from warnings import deprecated
|
||||
else:
|
||||
from typing_extensions import deprecated
|
||||
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@deprecated(
|
||||
"The AzureTextCompletion class is deprecated and will be removed after 01/01/2026. "
|
||||
"There won't be a replacement because all text completion models on Azure OpenAI "
|
||||
"have retired. Please migrate to chat completion models before the deprecation date."
|
||||
)
|
||||
class AzureTextCompletion(AzureOpenAIConfigBase, OpenAITextCompletionBase):
|
||||
"""Azure Text Completion class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureTextCompletion service.
|
||||
|
||||
Args:
|
||||
service_id: The service ID for the Azure deployment. (Optional)
|
||||
api_key (str | None): The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name (str | None): The optional deployment. If provided, will override the value
|
||||
(text_deployment_name) in the env vars or .env file.
|
||||
endpoint (str | None): The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url (str | None): The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version (str | None): The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure Active Directory token. (Optional)
|
||||
ad_token_provider: The Azure Active Directory token provider. (Optional)
|
||||
token_endpoint: The Azure Active Directory token endpoint. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (Optional[AsyncAzureOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
credential (TokenCredential): The credential to use for authentication. (Optional)
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
env_file_path=env_file_path,
|
||||
text_deployment_name=deployment_name,
|
||||
endpoint=endpoint,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError(f"Invalid settings: {ex}") from ex
|
||||
if not azure_openai_settings.text_deployment_name:
|
||||
raise ServiceInitializationError("The Azure Text deployment name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.text_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.TEXT,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, settings: dict[str, Any]) -> "AzureTextCompletion":
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: deployment_name, endpoint, api_key
|
||||
and optionally: api_version, ad_auth
|
||||
"""
|
||||
return AzureTextCompletion(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding_base import OpenAITextEmbeddingBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@experimental
|
||||
class AzureTextEmbedding(AzureOpenAIConfigBase, OpenAITextEmbeddingBase):
|
||||
"""Azure Text Embedding class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureTextEmbedding service.
|
||||
|
||||
service_id: The service ID. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(text_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure AD token for authentication. (Optional)
|
||||
ad_token_provider: Whether to use Azure Active Directory authentication.
|
||||
(Optional) The default value is False.
|
||||
token_endpoint: The Azure AD token endpoint. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (Optional[AsyncAzureOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
credential (TokenCredential): The credential to use for authentication.
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
env_file_path=env_file_path,
|
||||
api_key=api_key,
|
||||
embedding_deployment_name=deployment_name,
|
||||
endpoint=endpoint,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
|
||||
if not azure_openai_settings.embedding_deployment_name:
|
||||
raise ServiceInitializationError("The Azure OpenAI embedding deployment name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.embedding_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.EMBEDDING,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, settings: dict[str, Any]) -> "AzureTextEmbedding":
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: deployment_name, endpoint, api_key
|
||||
and optionally: api_version, ad_auth
|
||||
"""
|
||||
return AzureTextEmbedding(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio_base import OpenAITextToAudioBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="AzureTextToAudio")
|
||||
|
||||
|
||||
class AzureTextToAudio(AzureOpenAIConfigBase, OpenAITextToAudioBase):
|
||||
"""Azure text to audio service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = "2024-10-01-preview",
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureTextToAudio service.
|
||||
|
||||
Args:
|
||||
service_id: The service ID. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(text_to_audio_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file. Default is "2024-10-01-preview".
|
||||
ad_token: The Azure AD token for authentication. (Optional)
|
||||
ad_token_provider: Azure AD Token provider. (Optional)
|
||||
token_endpoint: The Azure AD token endpoint. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
api_key=api_key,
|
||||
text_to_audio_deployment_name=deployment_name,
|
||||
endpoint=endpoint,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
|
||||
if not azure_openai_settings.text_to_audio_deployment_name:
|
||||
raise ServiceInitializationError("The Azure OpenAI text to audio deployment name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.text_to_audio_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.TEXT_TO_AUDIO,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: deployment_name, endpoint, api_key
|
||||
and optionally: api_version, ad_auth
|
||||
"""
|
||||
return cls(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from azure.core.credentials import TokenCredential
|
||||
from openai import AsyncAzureOpenAI
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image_base import OpenAITextToImageBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="AzureTextToImage")
|
||||
|
||||
|
||||
class AzureTextToImage(AzureOpenAIConfigBase, OpenAITextToImageBase):
|
||||
"""Azure Text to Image service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
deployment_name: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
ad_token: str | None = None,
|
||||
ad_token_provider: AsyncAzureADTokenProvider | None = None,
|
||||
token_endpoint: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncAzureOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
credential: TokenCredential | None = None,
|
||||
) -> None:
|
||||
"""Initialize an AzureTextToImage service.
|
||||
|
||||
Args:
|
||||
service_id: The service ID. (Optional)
|
||||
api_key: The optional api key. If provided, will override the value in the
|
||||
env vars or .env file.
|
||||
deployment_name: The optional deployment. If provided, will override the value
|
||||
(text_to_image_deployment_name) in the env vars or .env file.
|
||||
endpoint: The optional deployment endpoint. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
base_url: The optional deployment base_url. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
api_version: The optional deployment api version. If provided will override the value
|
||||
in the env vars or .env file.
|
||||
ad_token: The Azure AD token for authentication. (Optional)
|
||||
ad_token_provider: Azure AD Token provider. (Optional)
|
||||
token_endpoint: The Azure AD token endpoint. (Optional)
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
credential: The credential to use for authentication. (Optional)
|
||||
"""
|
||||
try:
|
||||
azure_openai_settings = AzureOpenAISettings(
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
api_key=api_key,
|
||||
text_to_image_deployment_name=deployment_name,
|
||||
endpoint=endpoint,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
token_endpoint=token_endpoint,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
|
||||
if not azure_openai_settings.text_to_image_deployment_name:
|
||||
raise ServiceInitializationError("The Azure OpenAI text to image deployment name is required.")
|
||||
|
||||
super().__init__(
|
||||
deployment_name=azure_openai_settings.text_to_image_deployment_name,
|
||||
endpoint=azure_openai_settings.endpoint,
|
||||
base_url=azure_openai_settings.base_url,
|
||||
api_version=azure_openai_settings.api_version,
|
||||
service_id=service_id,
|
||||
api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
|
||||
ad_token=ad_token,
|
||||
ad_token_provider=ad_token_provider,
|
||||
token_endpoint=azure_openai_settings.token_endpoint,
|
||||
default_headers=default_headers,
|
||||
ai_model_type=OpenAIModelTypes.TEXT_TO_IMAGE,
|
||||
client=async_client,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Azure OpenAI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
should contain keys: deployment_name, endpoint, api_key
|
||||
and optionally: api_version, ad_auth
|
||||
"""
|
||||
return cls(
|
||||
service_id=settings.get("service_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
deployment_name=settings.get("deployment_name"),
|
||||
endpoint=settings.get("endpoint"),
|
||||
base_url=settings.get("base_url"),
|
||||
api_version=settings.get("api_version"),
|
||||
ad_token=settings.get("ad_token"),
|
||||
ad_token_provider=settings.get("ad_token_provider"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text_base import OpenAIAudioToTextBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="OpenAIAudioToText")
|
||||
|
||||
|
||||
class OpenAIAudioToText(OpenAIConfigBase, OpenAIAudioToTextBase):
|
||||
"""OpenAI Text to Image service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OpenAIAudioToText class.
|
||||
|
||||
Args:
|
||||
ai_model_id: OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id: Service ID tied to the execution settings.
|
||||
api_key: The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id: The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as
|
||||
a fallback to environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
audio_to_text_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.audio_to_text_model_id:
|
||||
raise ServiceInitializationError("The OpenAI audio to text model ID is required.")
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.audio_to_text_model_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
ai_model_type=OpenAIModelTypes.AUDIO_TO_TEXT,
|
||||
org_id=openai_settings.org_id,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
return cls(
|
||||
ai_model_id=settings.get("ai_model_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
org_id=settings.get("org_id"),
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers", {}),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from openai.types.audio import Transcription
|
||||
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from semantic_kernel.connectors.ai.audio_to_text_client_base import AudioToTextClientBase
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_audio_to_text_execution_settings import (
|
||||
OpenAIAudioToTextExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents import AudioContent, TextContent
|
||||
|
||||
|
||||
class OpenAIAudioToTextBase(OpenAIHandler, AudioToTextClientBase):
|
||||
"""OpenAI audio to text client."""
|
||||
|
||||
@override
|
||||
async def get_text_contents(
|
||||
self,
|
||||
audio_content: AudioContent,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[TextContent]:
|
||||
if not settings:
|
||||
settings = OpenAIAudioToTextExecutionSettings(ai_model_id=self.ai_model_id)
|
||||
else:
|
||||
if not isinstance(settings, OpenAIAudioToTextExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
|
||||
assert isinstance(settings, OpenAIAudioToTextExecutionSettings) # nosec
|
||||
|
||||
if settings.ai_model_id is None:
|
||||
settings.ai_model_id = self.ai_model_id
|
||||
|
||||
if not isinstance(audio_content.uri, str):
|
||||
raise ServiceInvalidRequestError("Audio content uri must be a string to a local file.")
|
||||
|
||||
settings.filename = audio_content.uri
|
||||
|
||||
response = await self._send_request(settings)
|
||||
assert isinstance(response, Transcription) # nosec
|
||||
|
||||
return [
|
||||
TextContent(
|
||||
ai_model_id=settings.ai_model_id,
|
||||
text=response.text,
|
||||
inner_content=response,
|
||||
)
|
||||
]
|
||||
|
||||
def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
|
||||
"""Get the request settings class."""
|
||||
return OpenAIAudioToTextExecutionSettings
|
||||
@@ -0,0 +1,91 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base import OpenAIChatCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenAIChatCompletion(OpenAIConfigBase, OpenAIChatCompletionBase, OpenAITextCompletionBase):
|
||||
"""OpenAI Chat completion class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
instruction_role: str | None = None,
|
||||
) -> None:
|
||||
"""Initialize an OpenAIChatCompletion service.
|
||||
|
||||
Args:
|
||||
ai_model_id (str): OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id (str | None): Service ID tied to the execution settings.
|
||||
api_key (str | None): The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id (str | None): The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback
|
||||
to environment variables. (Optional)
|
||||
env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
|
||||
instruction_role (str | None): The role to use for 'instruction' messages, for example,
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
chat_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
|
||||
if not async_client and not openai_settings.api_key:
|
||||
raise ServiceInitializationError("The OpenAI API key is required.")
|
||||
if not openai_settings.chat_model_id:
|
||||
raise ServiceInitializationError("The OpenAI model ID is required.")
|
||||
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.chat_model_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
org_id=openai_settings.org_id,
|
||||
service_id=service_id,
|
||||
ai_model_type=OpenAIModelTypes.CHAT,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
instruction_role=instruction_role,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, settings: dict[str, Any]) -> "OpenAIChatCompletion":
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
return OpenAIChatCompletion(
|
||||
ai_model_id=settings["ai_model_id"],
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
)
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from openai import AsyncStream
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, ChoiceDeltaFunctionCall, ChoiceDeltaToolCall
|
||||
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
|
||||
from openai.types.chat.chat_completion_message import FunctionCall
|
||||
from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall
|
||||
from typing_extensions import deprecated
|
||||
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
|
||||
from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration
|
||||
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior, FunctionChoiceType
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
|
||||
OpenAIChatPromptExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.contents.annotation_content import AnnotationContent
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.file_reference_content import FileReferenceContent
|
||||
from semantic_kernel.contents.function_call_content import FunctionCallContent
|
||||
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.contents.utils.author_role import AuthorRole
|
||||
from semantic_kernel.contents.utils.finish_reason import FinishReason
|
||||
from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError, ServiceInvalidResponseError
|
||||
from semantic_kernel.filters.auto_function_invocation.auto_function_invocation_context import (
|
||||
AutoFunctionInvocationContext,
|
||||
)
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
trace_chat_completion,
|
||||
trace_streaming_chat_completion,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
|
||||
|
||||
class OpenAIChatCompletionBase(OpenAIHandler, ChatCompletionClientBase):
|
||||
"""OpenAI Chat completion class."""
|
||||
|
||||
MODEL_PROVIDER_NAME: ClassVar[str] = "openai"
|
||||
SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True
|
||||
|
||||
# region Overriding base class methods
|
||||
# most of the methods are overridden from the ChatCompletionClientBase class, otherwise it is mentioned
|
||||
|
||||
# Override from AIServiceClientBase
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
return OpenAIChatPromptExecutionSettings
|
||||
|
||||
# Override from AIServiceClientBase
|
||||
@override
|
||||
def service_url(self) -> str | None:
|
||||
return str(self.client.base_url)
|
||||
|
||||
@override
|
||||
@trace_chat_completion(MODEL_PROVIDER_NAME)
|
||||
async def _inner_get_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> list["ChatMessageContent"]:
|
||||
if not isinstance(settings, OpenAIChatPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OpenAIChatPromptExecutionSettings) # nosec
|
||||
|
||||
settings.stream = False
|
||||
settings.messages = self._prepare_chat_history_for_request(chat_history)
|
||||
settings.ai_model_id = settings.ai_model_id or self.ai_model_id
|
||||
|
||||
response = await self._send_request(settings)
|
||||
assert isinstance(response, ChatCompletion) # nosec
|
||||
response_metadata = self._get_metadata_from_chat_response(response)
|
||||
return [self._create_chat_message_content(response, choice, response_metadata) for choice in response.choices]
|
||||
|
||||
@override
|
||||
@trace_streaming_chat_completion(MODEL_PROVIDER_NAME)
|
||||
async def _inner_get_streaming_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
function_invoke_attempt: int = 0,
|
||||
) -> AsyncGenerator[list["StreamingChatMessageContent"], Any]:
|
||||
if not isinstance(settings, OpenAIChatPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OpenAIChatPromptExecutionSettings) # nosec
|
||||
|
||||
settings.stream = True
|
||||
settings.stream_options = {"include_usage": True}
|
||||
settings.messages = self._prepare_chat_history_for_request(chat_history)
|
||||
settings.ai_model_id = settings.ai_model_id or self.ai_model_id
|
||||
|
||||
response = await self._send_request(settings)
|
||||
if not isinstance(response, AsyncStream):
|
||||
raise ServiceInvalidResponseError("Expected an AsyncStream[ChatCompletionChunk] response.")
|
||||
async for chunk in response:
|
||||
if len(chunk.choices) == 0 and chunk.usage is None:
|
||||
continue
|
||||
|
||||
assert isinstance(chunk, ChatCompletionChunk) # nosec
|
||||
chunk_metadata = self._get_metadata_from_streaming_chat_response(chunk)
|
||||
if (not chunk.choices or len(chunk.choices) == 0) and chunk.usage is not None:
|
||||
# Usage is contained in the last chunk where the choices are empty
|
||||
# We are duplicating the usage metadata to all the choices in the response
|
||||
yield [
|
||||
StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
content="",
|
||||
choice_index=i,
|
||||
inner_content=chunk,
|
||||
ai_model_id=settings.ai_model_id,
|
||||
metadata=chunk_metadata,
|
||||
function_invoke_attempt=function_invoke_attempt,
|
||||
)
|
||||
for i in range(settings.number_of_responses or 1)
|
||||
]
|
||||
else:
|
||||
yield [
|
||||
self._create_streaming_chat_message_content(chunk, choice, chunk_metadata, function_invoke_attempt)
|
||||
for choice in chunk.choices
|
||||
]
|
||||
|
||||
@override
|
||||
def _verify_function_choice_settings(self, settings: "PromptExecutionSettings") -> None:
|
||||
if not isinstance(settings, OpenAIChatPromptExecutionSettings):
|
||||
raise ServiceInvalidExecutionSettingsError("The settings must be an OpenAIChatPromptExecutionSettings.")
|
||||
if settings.number_of_responses is not None and settings.number_of_responses > 1:
|
||||
raise ServiceInvalidExecutionSettingsError(
|
||||
"Auto-invocation of tool calls may only be used with a "
|
||||
"OpenAIChatPromptExecutions.number_of_responses of 1."
|
||||
)
|
||||
|
||||
@override
|
||||
def _update_function_choice_settings_callback(
|
||||
self,
|
||||
) -> Callable[["FunctionCallChoiceConfiguration", "PromptExecutionSettings", FunctionChoiceType], None]:
|
||||
return update_settings_from_function_call_configuration
|
||||
|
||||
@override
|
||||
def _reset_function_choice_settings(self, settings: "PromptExecutionSettings") -> None:
|
||||
if hasattr(settings, "tool_choice"):
|
||||
settings.tool_choice = None
|
||||
if hasattr(settings, "tools"):
|
||||
settings.tools = None
|
||||
|
||||
# endregion
|
||||
|
||||
# region content creation
|
||||
|
||||
def _create_chat_message_content(
|
||||
self, response: ChatCompletion, choice: Choice, response_metadata: dict[str, Any]
|
||||
) -> "ChatMessageContent":
|
||||
"""Create a chat message content object from a choice."""
|
||||
metadata = self._get_metadata_from_chat_choice(choice)
|
||||
metadata.update(response_metadata)
|
||||
|
||||
items: list[Any] = self._get_tool_calls_from_chat_choice(choice)
|
||||
items.extend(self._get_function_call_from_chat_choice(choice))
|
||||
if choice.message.content:
|
||||
items.append(TextContent(text=choice.message.content))
|
||||
elif hasattr(choice.message, "refusal") and choice.message.refusal:
|
||||
items.append(TextContent(text=choice.message.refusal))
|
||||
|
||||
return ChatMessageContent(
|
||||
inner_content=response,
|
||||
ai_model_id=self.ai_model_id,
|
||||
metadata=metadata,
|
||||
role=AuthorRole(choice.message.role),
|
||||
items=items,
|
||||
finish_reason=(FinishReason(choice.finish_reason) if choice.finish_reason else None),
|
||||
)
|
||||
|
||||
def _create_streaming_chat_message_content(
|
||||
self,
|
||||
chunk: ChatCompletionChunk,
|
||||
choice: ChunkChoice,
|
||||
chunk_metadata: dict[str, Any],
|
||||
function_invoke_attempt: int,
|
||||
) -> StreamingChatMessageContent:
|
||||
"""Create a streaming chat message content object from a choice."""
|
||||
metadata = self._get_metadata_from_chat_choice(choice)
|
||||
metadata.update(chunk_metadata)
|
||||
|
||||
items: list[Any] = self._get_tool_calls_from_chat_choice(choice)
|
||||
items.extend(self._get_function_call_from_chat_choice(choice))
|
||||
if choice.delta and choice.delta.content is not None:
|
||||
items.append(StreamingTextContent(choice_index=choice.index, text=choice.delta.content))
|
||||
return StreamingChatMessageContent(
|
||||
choice_index=choice.index,
|
||||
inner_content=chunk,
|
||||
ai_model_id=self.ai_model_id,
|
||||
metadata=metadata,
|
||||
role=(AuthorRole(choice.delta.role) if choice.delta and choice.delta.role else AuthorRole.ASSISTANT),
|
||||
finish_reason=(FinishReason(choice.finish_reason) if choice.finish_reason else None),
|
||||
items=items,
|
||||
function_invoke_attempt=function_invoke_attempt,
|
||||
)
|
||||
|
||||
def _get_metadata_from_chat_response(self, response: ChatCompletion) -> dict[str, Any]:
|
||||
"""Get metadata from a chat response."""
|
||||
return {
|
||||
"id": response.id,
|
||||
"created": response.created,
|
||||
"system_fingerprint": response.system_fingerprint,
|
||||
"usage": CompletionUsage.from_openai(response.usage) if response.usage is not None else None,
|
||||
}
|
||||
|
||||
def _get_metadata_from_streaming_chat_response(self, response: ChatCompletionChunk) -> dict[str, Any]:
|
||||
"""Get metadata from a streaming chat response."""
|
||||
return {
|
||||
"id": response.id,
|
||||
"created": response.created,
|
||||
"system_fingerprint": response.system_fingerprint,
|
||||
"usage": CompletionUsage.from_openai(response.usage) if response.usage is not None else None,
|
||||
}
|
||||
|
||||
def _get_metadata_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[str, Any]:
|
||||
"""Get metadata from a chat choice."""
|
||||
return {
|
||||
"logprobs": getattr(choice, "logprobs", None),
|
||||
}
|
||||
|
||||
def _get_tool_calls_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]:
|
||||
"""Get tool calls from a chat choice."""
|
||||
content = choice.message if isinstance(choice, Choice) else choice.delta
|
||||
if content and (tool_calls := getattr(content, "tool_calls", None)) is not None:
|
||||
return [
|
||||
FunctionCallContent(
|
||||
id=tool.id,
|
||||
index=getattr(tool, "index", None),
|
||||
name=tool.function.name,
|
||||
arguments=tool.function.arguments,
|
||||
)
|
||||
for tool in cast(list[ChatCompletionMessageToolCall] | list[ChoiceDeltaToolCall], tool_calls)
|
||||
if tool.function is not None
|
||||
]
|
||||
# When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
|
||||
return []
|
||||
|
||||
def _get_function_call_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]:
|
||||
"""Get a function call from a chat choice."""
|
||||
content = choice.message if isinstance(choice, Choice) else choice.delta
|
||||
if content and (function_call := getattr(content, "function_call", None)) is not None:
|
||||
function_call = cast(FunctionCall | ChoiceDeltaFunctionCall, function_call)
|
||||
return [
|
||||
FunctionCallContent(
|
||||
id="legacy_function_call", name=function_call.name, arguments=function_call.arguments
|
||||
)
|
||||
]
|
||||
# When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
|
||||
return []
|
||||
|
||||
def _prepare_chat_history_for_request(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
role_key: str = "role",
|
||||
content_key: str = "content",
|
||||
) -> Any:
|
||||
"""Prepare the chat history for a request.
|
||||
|
||||
Allowing customization of the key names for role/author, and optionally overriding the role.
|
||||
|
||||
ChatRole.TOOL messages need to be formatted different than system/user/assistant messages:
|
||||
They require a "tool_call_id" and (function) "name" key, and the "metadata" key should
|
||||
be removed. The "encoding" key should also be removed.
|
||||
|
||||
Override this method to customize the formatting of the chat history for a request.
|
||||
|
||||
Args:
|
||||
chat_history (ChatHistory): The chat history to prepare.
|
||||
role_key (str): The key name for the role/author.
|
||||
content_key (str): The key name for the content/message.
|
||||
|
||||
Returns:
|
||||
prepared_chat_history (Any): The prepared chat history for a request.
|
||||
"""
|
||||
return [
|
||||
{
|
||||
**message.to_dict(role_key=role_key, content_key=content_key),
|
||||
role_key: "developer"
|
||||
if self.instruction_role == "developer" and message.to_dict(role_key=role_key)[role_key] == "system"
|
||||
else message.to_dict(role_key=role_key)[role_key],
|
||||
}
|
||||
for message in chat_history.messages
|
||||
if not isinstance(message, (AnnotationContent, FileReferenceContent))
|
||||
]
|
||||
|
||||
# endregion
|
||||
|
||||
# region function calling
|
||||
@deprecated("Use `invoke_function_call` from the kernel instead with `FunctionChoiceBehavior`.")
|
||||
async def _process_function_call(
|
||||
self,
|
||||
function_call: FunctionCallContent,
|
||||
chat_history: ChatHistory,
|
||||
kernel: "Kernel",
|
||||
arguments: "KernelArguments | None",
|
||||
function_call_count: int,
|
||||
request_index: int,
|
||||
function_call_behavior: FunctionChoiceBehavior,
|
||||
) -> "AutoFunctionInvocationContext | None":
|
||||
"""Processes the tool calls in the result and update the chat history."""
|
||||
return await kernel.invoke_function_call(
|
||||
function_call=function_call,
|
||||
chat_history=chat_history,
|
||||
arguments=arguments,
|
||||
function_call_count=function_call_count,
|
||||
request_index=request_index,
|
||||
function_behavior=function_call_behavior,
|
||||
)
|
||||
|
||||
# endregion
|
||||
@@ -0,0 +1,106 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from copy import copy
|
||||
from typing import Any
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ConfigDict, Field, validate_call
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.const import USER_AGENT
|
||||
from semantic_kernel.exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenAIConfigBase(OpenAIHandler):
|
||||
"""Internal class for configuring a connection to an OpenAI service."""
|
||||
|
||||
@validate_call(config=ConfigDict(arbitrary_types_allowed=True))
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str = Field(min_length=1),
|
||||
api_key: str | None = Field(min_length=1),
|
||||
ai_model_type: OpenAIModelTypes | None = OpenAIModelTypes.CHAT,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
client: AsyncOpenAI | None = None,
|
||||
instruction_role: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize a client for OpenAI services.
|
||||
|
||||
This constructor sets up a client to interact with OpenAI's API, allowing for
|
||||
different types of AI model interactions, like chat or text completion.
|
||||
|
||||
Args:
|
||||
ai_model_id (str): OpenAI model identifier. Must be non-empty.
|
||||
Default to a preset value.
|
||||
api_key (str): OpenAI API key for authentication.
|
||||
Must be non-empty. (Optional)
|
||||
ai_model_type (OpenAIModelTypes): The type of OpenAI
|
||||
model to interact with. Defaults to CHAT.
|
||||
org_id (str): OpenAI organization ID. This is optional
|
||||
unless the account belongs to multiple organizations.
|
||||
service_id (str): OpenAI service ID. This is optional.
|
||||
default_headers (Mapping[str, str]): Default headers
|
||||
for HTTP requests. (Optional)
|
||||
client (AsyncOpenAI): An existing OpenAI client, optional.
|
||||
instruction_role (str): The role to use for 'instruction'
|
||||
messages, for example, summarization prompts could use `developer` or `system`. (Optional)
|
||||
kwargs: Additional keyword arguments.
|
||||
|
||||
"""
|
||||
# Merge APP_INFO into the headers if it exists
|
||||
merged_headers = dict(copy(default_headers)) if default_headers else {}
|
||||
if APP_INFO:
|
||||
merged_headers.update(APP_INFO)
|
||||
merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers)
|
||||
|
||||
if not client:
|
||||
if not api_key:
|
||||
raise ServiceInitializationError("Please provide an api_key")
|
||||
client = AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
organization=org_id,
|
||||
default_headers=merged_headers,
|
||||
)
|
||||
args = {
|
||||
"ai_model_id": ai_model_id,
|
||||
"client": client,
|
||||
"ai_model_type": ai_model_type,
|
||||
}
|
||||
if service_id:
|
||||
args["service_id"] = service_id
|
||||
if instruction_role:
|
||||
args["instruction_role"] = instruction_role
|
||||
super().__init__(**args, **kwargs)
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
"""Create a dict of the service settings."""
|
||||
client_settings = {
|
||||
"api_key": self.client.api_key,
|
||||
"default_headers": {k: v for k, v in self.client.default_headers.items() if k != USER_AGENT},
|
||||
}
|
||||
if self.client.organization:
|
||||
client_settings["org_id"] = self.client.organization
|
||||
base = self.model_dump(
|
||||
exclude={
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"api_type",
|
||||
"ai_model_type",
|
||||
"service_id",
|
||||
"client",
|
||||
},
|
||||
by_alias=True,
|
||||
exclude_none=True,
|
||||
)
|
||||
base.update(client_settings)
|
||||
return base
|
||||
@@ -0,0 +1,235 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from abc import ABC
|
||||
from typing import Any, Union
|
||||
|
||||
from openai import AsyncOpenAI, AsyncStream, BadRequestError, _legacy_response
|
||||
from openai._types import FileTypes, Omit, omit
|
||||
from openai.lib._parsing._completions import type_to_response_format_param
|
||||
from openai.types import Completion, CreateEmbeddingResponse
|
||||
from openai.types.audio import Transcription
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
from openai.types.images_response import ImagesResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai import (
|
||||
OpenAIAudioToTextExecutionSettings,
|
||||
OpenAIChatPromptExecutionSettings,
|
||||
OpenAIEmbeddingPromptExecutionSettings,
|
||||
OpenAIPromptExecutionSettings,
|
||||
OpenAITextToAudioExecutionSettings,
|
||||
OpenAITextToImageExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import ContentFilterAIException
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.utils.structured_output_schema import generate_structured_output_response_format_schema
|
||||
from semantic_kernel.exceptions import ServiceResponseException
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
|
||||
from semantic_kernel.kernel_pydantic import KernelBaseModel
|
||||
from semantic_kernel.schema.kernel_json_schema_builder import KernelJsonSchemaBuilder
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
RESPONSE_TYPE = Union[
|
||||
ChatCompletion,
|
||||
Completion,
|
||||
AsyncStream[ChatCompletionChunk],
|
||||
AsyncStream[Completion],
|
||||
list[Any],
|
||||
ImagesResponse,
|
||||
Transcription,
|
||||
_legacy_response.HttpxBinaryResponseContent,
|
||||
]
|
||||
|
||||
|
||||
class OpenAIHandler(KernelBaseModel, ABC):
|
||||
"""Internal class for calls to OpenAI API's."""
|
||||
|
||||
client: AsyncOpenAI
|
||||
ai_model_type: OpenAIModelTypes = OpenAIModelTypes.CHAT
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
|
||||
async def _send_request(self, settings: PromptExecutionSettings) -> RESPONSE_TYPE:
|
||||
"""Send a request to the OpenAI API."""
|
||||
if self.ai_model_type == OpenAIModelTypes.TEXT or self.ai_model_type == OpenAIModelTypes.CHAT:
|
||||
assert isinstance(settings, OpenAIPromptExecutionSettings) # nosec
|
||||
return await self._send_completion_request(settings)
|
||||
if self.ai_model_type == OpenAIModelTypes.EMBEDDING:
|
||||
assert isinstance(settings, OpenAIEmbeddingPromptExecutionSettings) # nosec
|
||||
return await self._send_embedding_request(settings)
|
||||
if self.ai_model_type == OpenAIModelTypes.TEXT_TO_IMAGE:
|
||||
assert isinstance(settings, OpenAITextToImageExecutionSettings) # nosec
|
||||
return await self._send_text_to_image_request(settings)
|
||||
if self.ai_model_type == OpenAIModelTypes.AUDIO_TO_TEXT:
|
||||
assert isinstance(settings, OpenAIAudioToTextExecutionSettings) # nosec
|
||||
return await self._send_audio_to_text_request(settings)
|
||||
if self.ai_model_type == OpenAIModelTypes.TEXT_TO_AUDIO:
|
||||
assert isinstance(settings, OpenAITextToAudioExecutionSettings) # nosec
|
||||
return await self._send_text_to_audio_request(settings)
|
||||
|
||||
raise NotImplementedError(f"Model type {self.ai_model_type} is not supported")
|
||||
|
||||
async def _send_completion_request(
|
||||
self,
|
||||
settings: OpenAIPromptExecutionSettings,
|
||||
) -> ChatCompletion | Completion | AsyncStream[ChatCompletionChunk] | AsyncStream[Completion]:
|
||||
"""Execute the appropriate call to OpenAI models."""
|
||||
try:
|
||||
settings_dict = settings.prepare_settings_dict()
|
||||
if self.ai_model_type == OpenAIModelTypes.CHAT:
|
||||
assert isinstance(settings, OpenAIChatPromptExecutionSettings) # nosec
|
||||
self._handle_structured_output(settings, settings_dict)
|
||||
if settings.tools is None:
|
||||
settings_dict.pop("parallel_tool_calls", None)
|
||||
response = await self.client.chat.completions.create(**settings_dict)
|
||||
else:
|
||||
response = await self.client.completions.create(**settings_dict)
|
||||
|
||||
self.store_usage(response)
|
||||
return response
|
||||
except BadRequestError as ex:
|
||||
if ex.code == "content_filter":
|
||||
raise ContentFilterAIException(
|
||||
f"{type(self)} service encountered a content error",
|
||||
ex,
|
||||
) from ex
|
||||
raise ServiceResponseException(
|
||||
f"{type(self)} service failed to complete the prompt",
|
||||
ex,
|
||||
) from ex
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(
|
||||
f"{type(self)} service failed to complete the prompt",
|
||||
ex,
|
||||
) from ex
|
||||
|
||||
async def _send_embedding_request(self, settings: OpenAIEmbeddingPromptExecutionSettings) -> list[Any]:
|
||||
"""Send a request to the OpenAI embeddings endpoint."""
|
||||
try:
|
||||
response = await self.client.embeddings.create(**settings.prepare_settings_dict())
|
||||
|
||||
self.store_usage(response)
|
||||
return [x.embedding for x in response.data]
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(
|
||||
f"{type(self)} service failed to generate embeddings",
|
||||
ex,
|
||||
) from ex
|
||||
|
||||
async def _send_text_to_image_request(self, settings: OpenAITextToImageExecutionSettings) -> ImagesResponse:
|
||||
"""Send a request to the OpenAI text to image endpoint."""
|
||||
try:
|
||||
response: ImagesResponse = await self.client.images.generate(
|
||||
**settings.prepare_settings_dict(),
|
||||
)
|
||||
self.store_usage(response)
|
||||
return response
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(f"Failed to generate image: {ex}") from ex
|
||||
|
||||
async def _send_image_edit_request(
|
||||
self,
|
||||
image: list[FileTypes],
|
||||
settings: OpenAITextToImageExecutionSettings,
|
||||
mask: FileTypes | Omit = omit,
|
||||
) -> ImagesResponse:
|
||||
"""Send a request to the OpenAI image edit endpoint.
|
||||
|
||||
Args:
|
||||
image: List of image files to edit. Accepts file paths or bytes.
|
||||
settings: Image edit execution settings.
|
||||
mask: Optional mask image. Accepts file path or bytes.
|
||||
|
||||
Returns:
|
||||
ImagesResponse: The response from the image edit API.
|
||||
"""
|
||||
try:
|
||||
response: ImagesResponse = await self.client.images.edit(
|
||||
image=image,
|
||||
mask=mask, # type: ignore
|
||||
**settings.prepare_settings_dict(),
|
||||
)
|
||||
self.store_usage(response)
|
||||
return response
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(f"Failed to edit image: {ex}") from ex
|
||||
|
||||
async def _send_audio_to_text_request(self, settings: OpenAIAudioToTextExecutionSettings) -> Transcription:
|
||||
"""Send a request to the OpenAI audio to text endpoint."""
|
||||
if not settings.filename:
|
||||
raise ServiceInvalidRequestError("Audio file is required for audio to text service")
|
||||
|
||||
try:
|
||||
with open(settings.filename, "rb") as audio_file:
|
||||
return await self.client.audio.transcriptions.create(
|
||||
file=audio_file,
|
||||
**settings.prepare_settings_dict(),
|
||||
)
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(
|
||||
f"{type(self)} service failed to transcribe audio",
|
||||
ex,
|
||||
) from ex
|
||||
|
||||
async def _send_text_to_audio_request(
|
||||
self, settings: OpenAITextToAudioExecutionSettings
|
||||
) -> _legacy_response.HttpxBinaryResponseContent:
|
||||
"""Send a request to the OpenAI text to audio endpoint.
|
||||
|
||||
The OpenAI API returns the content of the generated audio file.
|
||||
"""
|
||||
try:
|
||||
return await self.client.audio.speech.create(
|
||||
**settings.prepare_settings_dict(),
|
||||
)
|
||||
except Exception as ex:
|
||||
raise ServiceResponseException(
|
||||
f"{type(self)} service failed to generate audio",
|
||||
ex,
|
||||
) from ex
|
||||
|
||||
def _handle_structured_output(
|
||||
self, request_settings: OpenAIChatPromptExecutionSettings, settings: dict[str, Any]
|
||||
) -> None:
|
||||
response_format = getattr(request_settings, "response_format", None)
|
||||
if getattr(request_settings, "structured_json_response", False) and response_format:
|
||||
# Case 1: response_format is a type and subclass of BaseModel
|
||||
if isinstance(response_format, type) and issubclass(response_format, BaseModel):
|
||||
settings["response_format"] = type_to_response_format_param(response_format)
|
||||
# Case 2: response_format is a type but not a subclass of BaseModel
|
||||
elif isinstance(response_format, type):
|
||||
generated_schema = KernelJsonSchemaBuilder.build(parameter_type=response_format, structured_output=True)
|
||||
assert generated_schema is not None # nosec
|
||||
settings["response_format"] = generate_structured_output_response_format_schema(
|
||||
name=response_format.__name__, schema=generated_schema
|
||||
)
|
||||
# Case 3: response_format is a dictionary, pass it without modification
|
||||
elif isinstance(response_format, dict):
|
||||
settings["response_format"] = response_format
|
||||
|
||||
def store_usage(
|
||||
self,
|
||||
response: ChatCompletion
|
||||
| Completion
|
||||
| AsyncStream[ChatCompletionChunk]
|
||||
| AsyncStream[Completion]
|
||||
| CreateEmbeddingResponse
|
||||
| ImagesResponse,
|
||||
):
|
||||
"""Store the usage information from the response."""
|
||||
if isinstance(response, ImagesResponse) and hasattr(response, "usage") and response.usage:
|
||||
logger.info(f"OpenAI image usage: {response.usage}")
|
||||
self.prompt_tokens += response.usage.input_tokens
|
||||
self.total_tokens += response.usage.total_tokens
|
||||
self.completion_tokens += response.usage.output_tokens
|
||||
return
|
||||
if not isinstance(response, AsyncStream) and not isinstance(response, ImagesResponse) and response.usage:
|
||||
logger.info(f"OpenAI usage: {response.usage}")
|
||||
self.prompt_tokens += response.usage.prompt_tokens
|
||||
self.total_tokens += response.usage.total_tokens
|
||||
if hasattr(response.usage, "completion_tokens"):
|
||||
self.completion_tokens += response.usage.completion_tokens # type: ignore
|
||||
@@ -0,0 +1,16 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class OpenAIModelTypes(Enum):
|
||||
"""OpenAI model types, can be text, chat or embedding."""
|
||||
|
||||
TEXT = "text"
|
||||
CHAT = "chat"
|
||||
EMBEDDING = "embedding"
|
||||
TEXT_TO_IMAGE = "text-to-image"
|
||||
AUDIO_TO_TEXT = "audio-to-text"
|
||||
TEXT_TO_AUDIO = "text-to-audio"
|
||||
REALTIME = "realtime"
|
||||
RESPONSE = "response"
|
||||
@@ -0,0 +1,190 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from numpy import ndarray
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import (
|
||||
ListenEvents,
|
||||
OpenAIRealtimeWebRTCBase,
|
||||
OpenAIRealtimeWebsocketBase,
|
||||
SendEvents,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiortc.mediastreams import MediaStreamTrack
|
||||
from numpy import ndarray
|
||||
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"ListenEvents",
|
||||
"OpenAIRealtimeWebRTC",
|
||||
"OpenAIRealtimeWebsocket",
|
||||
"SendEvents",
|
||||
]
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAIRealtimeWebRTC(OpenAIRealtimeWebRTCBase, OpenAIConfigBase):
|
||||
"""OpenAI Realtime service using WebRTC protocol."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_track: "MediaStreamTrack",
|
||||
audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize an OpenAIRealtime service.
|
||||
|
||||
Args:
|
||||
audio_track: The audio track to use for the service, only used by WebRTC.
|
||||
It can be any class that implements the AudioStreamTrack interface.
|
||||
audio_output_callback: The audio output callback, optional.
|
||||
This should be a coroutine, that takes a ndarray with audio as input.
|
||||
The goal of this function is to allow you to play the audio with the
|
||||
least amount of latency possible, because it is called first before further processing.
|
||||
It can also be set in the `receive` method.
|
||||
Even when passed, the audio content will still be
|
||||
added to the receiving queue.
|
||||
ai_model_id (str | None): OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id (str | None): Service ID tied to the execution settings.
|
||||
api_key (str | None): The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id (str | None): The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
|
||||
kwargs: Additional arguments.
|
||||
This can include:
|
||||
kernel (Kernel): the kernel to use for function calls
|
||||
plugins (list[object] or dict[str, object]): the plugins to use for function calls
|
||||
settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
|
||||
chat_history (ChatHistory): the chat history to use for the session
|
||||
Otherwise they can also be passed to the context manager.
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
realtime_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.realtime_model_id:
|
||||
raise ServiceInitializationError("The OpenAI realtime model ID is required.")
|
||||
if audio_track:
|
||||
kwargs["audio_track"] = audio_track
|
||||
super().__init__(
|
||||
audio_output_callback=audio_output_callback,
|
||||
ai_model_id=openai_settings.realtime_model_id,
|
||||
service_id=service_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
org_id=openai_settings.org_id,
|
||||
ai_model_type=OpenAIModelTypes.REALTIME,
|
||||
default_headers=default_headers,
|
||||
client=client,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# region Websocket
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAIRealtimeWebsocket(OpenAIRealtimeWebsocketBase, OpenAIConfigBase):
|
||||
"""OpenAI Realtime service using WebSocket protocol."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize an OpenAIRealtime service.
|
||||
|
||||
Args:
|
||||
audio_output_callback: The audio output callback, optional.
|
||||
This should be a coroutine, that takes a ndarray with audio as input.
|
||||
The goal of this function is to allow you to play the audio with the
|
||||
least amount of latency possible, because it is called first before further processing.
|
||||
It can also be set in the `receive` method.
|
||||
Even when passed, the audio content will still be
|
||||
added to the receiving queue.
|
||||
ai_model_id (str | None): OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id (str | None): Service ID tied to the execution settings.
|
||||
api_key (str | None): The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id (str | None): The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
|
||||
kwargs: Additional arguments.
|
||||
kwargs: Additional arguments.
|
||||
This can include:
|
||||
kernel (Kernel): the kernel to use for function calls
|
||||
plugins (list[object] or dict[str, object]): the plugins to use for function calls
|
||||
settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
|
||||
chat_history (ChatHistory): the chat history to use for the session
|
||||
Otherwise they can also be passed to the context manager.
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
realtime_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.realtime_model_id:
|
||||
raise ServiceInitializationError("The OpenAI realtime model ID is required.")
|
||||
super().__init__(
|
||||
audio_output_callback=audio_output_callback,
|
||||
ai_model_id=openai_settings.realtime_model_id,
|
||||
service_id=service_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
org_id=openai_settings.org_id,
|
||||
ai_model_type=OpenAIModelTypes.REALTIME,
|
||||
default_headers=default_headers,
|
||||
client=client,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,89 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenAITextCompletion(OpenAITextCompletionBase, OpenAIConfigBase):
|
||||
"""OpenAI Text Completion class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initialize an OpenAITextCompletion service.
|
||||
|
||||
Args:
|
||||
ai_model_id (str | None): OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id (str | None): Service ID tied to the execution settings.
|
||||
api_key (str | None): The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id (str | None): The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as a fallback to
|
||||
environment variables. (Optional)
|
||||
env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
text_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.text_model_id:
|
||||
raise ServiceInitializationError("The OpenAI text model ID is required.")
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.text_model_id,
|
||||
service_id=service_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
org_id=openai_settings.org_id,
|
||||
ai_model_type=OpenAIModelTypes.TEXT,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, settings: dict[str, Any]) -> "OpenAITextCompletion":
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
if "default_headers" in settings and isinstance(settings["default_headers"], str):
|
||||
settings["default_headers"] = json.loads(settings["default_headers"])
|
||||
return OpenAITextCompletion(
|
||||
ai_model_id=settings.get("ai_model_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
org_id=settings.get("org_id"),
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers"),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from openai import AsyncStream
|
||||
from openai.types import Completion as TextCompletion
|
||||
from openai.types import CompletionChoice as TextCompletionChoice
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
from openai.types.chat.chat_completion import Choice as ChatCompletionChoice
|
||||
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion_chunk import Choice as ChatCompletionChunkChoice
|
||||
|
||||
from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
|
||||
OpenAIChatPromptExecutionSettings,
|
||||
OpenAITextPromptExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
trace_streaming_text_completion,
|
||||
trace_text_completion,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OpenAITextCompletionBase(OpenAIHandler, TextCompletionClientBase):
|
||||
"""Base class for OpenAI text completion services."""
|
||||
|
||||
MODEL_PROVIDER_NAME: ClassVar[str] = "openai"
|
||||
|
||||
# region Overriding base class methods
|
||||
|
||||
# Override from AIServiceClientBase
|
||||
@override
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
return OpenAITextPromptExecutionSettings
|
||||
|
||||
# Override from AIServiceClientBase
|
||||
@override
|
||||
def service_url(self) -> str | None:
|
||||
return str(self.client.base_url)
|
||||
|
||||
@override
|
||||
@trace_text_completion(MODEL_PROVIDER_NAME)
|
||||
async def _inner_get_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> list["TextContent"]:
|
||||
if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)) # nosec
|
||||
|
||||
if isinstance(settings, OpenAITextPromptExecutionSettings):
|
||||
settings.prompt = prompt
|
||||
else:
|
||||
settings.messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
settings.ai_model_id = settings.ai_model_id or self.ai_model_id
|
||||
|
||||
response = await self._send_request(settings)
|
||||
assert isinstance(response, (TextCompletion, ChatCompletion)) # nosec
|
||||
|
||||
metadata = self._get_metadata_from_text_response(response)
|
||||
return [self._create_text_content(response, choice, metadata) for choice in response.choices]
|
||||
|
||||
@override
|
||||
@trace_streaming_text_completion(MODEL_PROVIDER_NAME)
|
||||
async def _inner_get_streaming_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> AsyncGenerator[list["StreamingTextContent"], Any]:
|
||||
if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)) # nosec
|
||||
|
||||
if isinstance(settings, OpenAITextPromptExecutionSettings):
|
||||
settings.prompt = prompt
|
||||
else:
|
||||
if not settings.messages:
|
||||
settings.messages = [{"role": "user", "content": prompt}]
|
||||
else:
|
||||
settings.messages.append({"role": "user", "content": prompt})
|
||||
|
||||
settings.ai_model_id = settings.ai_model_id or self.ai_model_id
|
||||
settings.stream = True
|
||||
|
||||
response = await self._send_request(settings)
|
||||
assert isinstance(response, AsyncStream) # nosec
|
||||
|
||||
async for chunk in response:
|
||||
if len(chunk.choices) == 0:
|
||||
continue
|
||||
assert isinstance(chunk, (TextCompletion, ChatCompletionChunk)) # nosec
|
||||
chunk_metadata = self._get_metadata_from_text_response(chunk)
|
||||
yield [self._create_streaming_text_content(chunk, choice, chunk_metadata) for choice in chunk.choices]
|
||||
|
||||
# endregion
|
||||
|
||||
def _create_text_content(
|
||||
self,
|
||||
response: TextCompletion | ChatCompletion,
|
||||
choice: TextCompletionChoice | ChatCompletionChoice,
|
||||
response_metadata: dict[str, Any],
|
||||
) -> "TextContent":
|
||||
"""Create a text content object from a choice."""
|
||||
choice_metadata = self._get_metadata_from_text_choice(choice)
|
||||
choice_metadata.update(response_metadata)
|
||||
text = choice.text if isinstance(choice, TextCompletionChoice) else choice.message.content
|
||||
return TextContent(
|
||||
inner_content=response,
|
||||
ai_model_id=self.ai_model_id,
|
||||
text=text or "",
|
||||
metadata=choice_metadata,
|
||||
)
|
||||
|
||||
def _create_streaming_text_content(
|
||||
self,
|
||||
chunk: TextCompletion | ChatCompletionChunk,
|
||||
choice: TextCompletionChoice | ChatCompletionChunkChoice,
|
||||
response_metadata: dict[str, Any],
|
||||
) -> "StreamingTextContent":
|
||||
"""Create a streaming text content object from a choice."""
|
||||
choice_metadata = self._get_metadata_from_text_choice(choice)
|
||||
choice_metadata.update(response_metadata)
|
||||
text = choice.text if isinstance(choice, TextCompletionChoice) else choice.delta.content
|
||||
return StreamingTextContent(
|
||||
choice_index=choice.index,
|
||||
inner_content=chunk,
|
||||
ai_model_id=self.ai_model_id,
|
||||
metadata=choice_metadata,
|
||||
text=text or "",
|
||||
)
|
||||
|
||||
def _get_metadata_from_text_response(
|
||||
self, response: TextCompletion | ChatCompletion | ChatCompletionChunk
|
||||
) -> dict[str, Any]:
|
||||
"""Get metadata from a response."""
|
||||
ret = {
|
||||
"id": response.id,
|
||||
"created": response.created,
|
||||
"system_fingerprint": response.system_fingerprint,
|
||||
}
|
||||
if response.usage is not None:
|
||||
ret["usage"] = CompletionUsage.from_openai(response.usage)
|
||||
return ret
|
||||
|
||||
def _get_metadata_from_text_choice(
|
||||
self, choice: TextCompletionChoice | ChatCompletionChoice | ChatCompletionChunkChoice
|
||||
) -> dict[str, Any]:
|
||||
"""Get metadata from a completion choice."""
|
||||
return {
|
||||
"logprobs": getattr(choice, "logprobs", None),
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding_base import OpenAITextEmbeddingBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
T_ = TypeVar("T_", bound="OpenAITextEmbedding")
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAITextEmbedding(OpenAIConfigBase, OpenAITextEmbeddingBase):
|
||||
"""OpenAI Text Embedding class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OpenAITextCompletion class.
|
||||
|
||||
Args:
|
||||
ai_model_id (str): OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id (str | None): Service ID tied to the execution settings.
|
||||
api_key (str | None): The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id (str | None): The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers (Mapping[str,str] | None): The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
|
||||
env_file_path (str | None): Use the environment settings file as
|
||||
a fallback to environment variables. (Optional)
|
||||
env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
embedding_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.embedding_model_id:
|
||||
raise ServiceInitializationError("The OpenAI embedding model ID is required.")
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.embedding_model_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
ai_model_type=OpenAIModelTypes.EMBEDDING,
|
||||
org_id=openai_settings.org_id,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
return cls(
|
||||
ai_model_id=settings.get("ai_model_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
org_id=settings.get("org_id"),
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers", {}),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from numpy import array, ndarray
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
|
||||
OpenAIEmbeddingPromptExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
|
||||
|
||||
@experimental
|
||||
class OpenAITextEmbeddingBase(OpenAIHandler, EmbeddingGeneratorBase):
|
||||
"""Base class for OpenAI text embedding services."""
|
||||
|
||||
@override
|
||||
async def generate_embeddings(
|
||||
self,
|
||||
texts: list[str],
|
||||
settings: "PromptExecutionSettings | None" = None,
|
||||
batch_size: int | None = None,
|
||||
**kwargs: Any,
|
||||
) -> ndarray:
|
||||
raw_embeddings = await self.generate_raw_embeddings(texts, settings, batch_size, **kwargs)
|
||||
return array(raw_embeddings)
|
||||
|
||||
@override
|
||||
async def generate_raw_embeddings(
|
||||
self,
|
||||
texts: list[str],
|
||||
settings: "PromptExecutionSettings | None" = None,
|
||||
batch_size: int | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Returns embeddings for the given texts in the unedited format.
|
||||
|
||||
Args:
|
||||
texts (List[str]): The texts to generate embeddings for.
|
||||
settings (PromptExecutionSettings): The settings to use for the request.
|
||||
batch_size (int): The batch size to use for the request.
|
||||
kwargs (Dict[str, Any]): Additional arguments to pass to the request.
|
||||
"""
|
||||
if not settings:
|
||||
settings = OpenAIEmbeddingPromptExecutionSettings(ai_model_id=self.ai_model_id)
|
||||
else:
|
||||
if not isinstance(settings, OpenAIEmbeddingPromptExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
assert isinstance(settings, OpenAIEmbeddingPromptExecutionSettings) # nosec
|
||||
if settings.ai_model_id is None:
|
||||
settings.ai_model_id = self.ai_model_id
|
||||
for key, value in kwargs.items():
|
||||
setattr(settings, key, value)
|
||||
raw_embeddings = []
|
||||
batch_size = batch_size or len(texts)
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
settings.input = batch
|
||||
raw_embedding = await self._send_request(settings=settings)
|
||||
assert isinstance(raw_embedding, list) # nosec
|
||||
raw_embeddings.extend(raw_embedding)
|
||||
return raw_embeddings
|
||||
|
||||
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
|
||||
"""Get the request settings class."""
|
||||
return OpenAIEmbeddingPromptExecutionSettings
|
||||
@@ -0,0 +1,85 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio_base import OpenAITextToAudioBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="OpenAITextToAudio")
|
||||
|
||||
|
||||
class OpenAITextToAudio(OpenAIConfigBase, OpenAITextToAudioBase):
|
||||
"""OpenAI Text to Image service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OpenAITextToAudio class.
|
||||
|
||||
Args:
|
||||
ai_model_id: OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id: Service ID tied to the execution settings.
|
||||
api_key: The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id: The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as
|
||||
a fallback to environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
text_to_audio_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.text_to_audio_model_id:
|
||||
raise ServiceInitializationError("The OpenAI text to audio model ID is required.")
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.text_to_audio_model_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
ai_model_type=OpenAIModelTypes.TEXT_TO_AUDIO,
|
||||
org_id=openai_settings.org_id,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
return cls(
|
||||
ai_model_id=settings.get("ai_model_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
org_id=settings.get("org_id"),
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers", {}),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from openai import _legacy_response
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_audio_execution_settings import (
|
||||
OpenAITextToAudioExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_to_audio_client_base import TextToAudioClientBase
|
||||
from semantic_kernel.contents.audio_content import AudioContent
|
||||
|
||||
|
||||
class OpenAITextToAudioBase(OpenAIHandler, TextToAudioClientBase):
|
||||
"""OpenAI text to audio client base class."""
|
||||
|
||||
@override
|
||||
async def get_audio_contents(
|
||||
self,
|
||||
text: str,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[AudioContent]:
|
||||
if not settings:
|
||||
settings = OpenAITextToAudioExecutionSettings(ai_model_id=self.ai_model_id)
|
||||
else:
|
||||
if not isinstance(settings, OpenAITextToAudioExecutionSettings):
|
||||
settings = self.get_prompt_execution_settings_from_settings(settings)
|
||||
|
||||
assert isinstance(settings, OpenAITextToAudioExecutionSettings) # nosec
|
||||
|
||||
if settings.ai_model_id is None:
|
||||
settings.ai_model_id = self.ai_model_id
|
||||
settings.input = text
|
||||
|
||||
response = await self._send_request(settings)
|
||||
assert isinstance(response, _legacy_response.HttpxBinaryResponseContent) # nosec
|
||||
|
||||
return [
|
||||
AudioContent(
|
||||
ai_model_id=settings.ai_model_id,
|
||||
data=response.read(),
|
||||
data_format="base64",
|
||||
)
|
||||
]
|
||||
|
||||
def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
|
||||
"""Get the request settings class."""
|
||||
return OpenAITextToAudioExecutionSettings
|
||||
@@ -0,0 +1,85 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import ValidationError
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image_base import OpenAITextToImageBase
|
||||
from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
|
||||
|
||||
T_ = TypeVar("T_", bound="OpenAITextToImage")
|
||||
|
||||
|
||||
class OpenAITextToImage(OpenAIConfigBase, OpenAITextToImageBase):
|
||||
"""OpenAI Text to Image service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ai_model_id: str | None = None,
|
||||
api_key: str | None = None,
|
||||
org_id: str | None = None,
|
||||
service_id: str | None = None,
|
||||
default_headers: Mapping[str, str] | None = None,
|
||||
async_client: AsyncOpenAI | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the OpenAITextToImage class.
|
||||
|
||||
Args:
|
||||
ai_model_id: OpenAI model name, see
|
||||
https://platform.openai.com/docs/models
|
||||
service_id: Service ID tied to the execution settings.
|
||||
api_key: The optional API key to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
org_id: The optional org ID to use. If provided will override,
|
||||
the env vars or .env file value.
|
||||
default_headers: The default headers mapping of string keys to
|
||||
string values for HTTP requests. (Optional)
|
||||
async_client: An existing client to use. (Optional)
|
||||
env_file_path: Use the environment settings file as
|
||||
a fallback to environment variables. (Optional)
|
||||
env_file_encoding: The encoding of the environment settings file. (Optional)
|
||||
"""
|
||||
try:
|
||||
openai_settings = OpenAISettings(
|
||||
api_key=api_key,
|
||||
org_id=org_id,
|
||||
text_to_image_model_id=ai_model_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
|
||||
if not openai_settings.text_to_image_model_id:
|
||||
raise ServiceInitializationError("The OpenAI text to image model ID is required.")
|
||||
super().__init__(
|
||||
ai_model_id=openai_settings.text_to_image_model_id,
|
||||
api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
|
||||
ai_model_type=OpenAIModelTypes.TEXT_TO_IMAGE,
|
||||
org_id=openai_settings.org_id,
|
||||
service_id=service_id,
|
||||
default_headers=default_headers,
|
||||
client=async_client,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
|
||||
"""Initialize an Open AI service from a dictionary of settings.
|
||||
|
||||
Args:
|
||||
settings: A dictionary of settings for the service.
|
||||
"""
|
||||
return cls(
|
||||
ai_model_id=settings.get("ai_model_id"),
|
||||
api_key=settings.get("api_key"),
|
||||
org_id=settings.get("org_id"),
|
||||
service_id=settings.get("service_id"),
|
||||
default_headers=settings.get("default_headers", {}),
|
||||
env_file_path=settings.get("env_file_path"),
|
||||
)
|
||||
@@ -0,0 +1,249 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pathlib import Path
|
||||
from typing import IO, Any
|
||||
from warnings import warn
|
||||
|
||||
from openai._types import FileTypes, Omit, omit
|
||||
from openai.types.images_response import ImagesResponse
|
||||
|
||||
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_image_execution_settings import (
|
||||
ImageSize,
|
||||
OpenAITextToImageExecutionSettings,
|
||||
)
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_to_image_client_base import TextToImageClientBase
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError, ServiceResponseException
|
||||
|
||||
|
||||
class OpenAITextToImageBase(OpenAIHandler, TextToImageClientBase):
|
||||
"""OpenAI text to image client."""
|
||||
|
||||
async def generate_image(
|
||||
self,
|
||||
description: str,
|
||||
width: int | None = None,
|
||||
height: int | None = None,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
**kwargs: Any,
|
||||
) -> bytes | str:
|
||||
"""Generate image from text.
|
||||
|
||||
Args:
|
||||
description: Description of the image.
|
||||
width: Deprecated, use settings instead.
|
||||
height: Deprecated, use settings instead.
|
||||
settings: Execution settings for the prompt.
|
||||
kwargs: Additional arguments, check the openai images.generate documentation for the supported arguments.
|
||||
|
||||
Returns:
|
||||
bytes | str: Image bytes or image URL.
|
||||
"""
|
||||
warn("generate_image is deprecated. Use generate_images.", DeprecationWarning, stacklevel=2)
|
||||
if not settings:
|
||||
settings = OpenAITextToImageExecutionSettings(**kwargs)
|
||||
if not isinstance(settings, OpenAITextToImageExecutionSettings):
|
||||
settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
|
||||
if width:
|
||||
warn("The 'width' argument is deprecated. Use 'settings.size' instead.", DeprecationWarning)
|
||||
if settings.size and not settings.size.width:
|
||||
settings.size.width = width
|
||||
if height:
|
||||
warn("The 'height' argument is deprecated. Use 'settings.size' instead.", DeprecationWarning)
|
||||
if settings.size and not settings.size.height:
|
||||
settings.size.height = height
|
||||
if not settings.size and width and height:
|
||||
settings.size = ImageSize(width=width, height=height)
|
||||
|
||||
if not settings.prompt:
|
||||
settings.prompt = description
|
||||
|
||||
if not settings.prompt:
|
||||
raise ServiceInvalidRequestError("Prompt is required.")
|
||||
|
||||
if not settings.ai_model_id:
|
||||
settings.ai_model_id = self.ai_model_id
|
||||
|
||||
response = await self._send_request(settings)
|
||||
|
||||
assert isinstance(response, ImagesResponse) # nosec
|
||||
if not response.data or not (response.data[0].url or response.data[0].b64_json):
|
||||
raise ServiceResponseException("Failed to generate image.")
|
||||
|
||||
return response.data[0].url or response.data[0].b64_json # type: ignore[return-value]
|
||||
|
||||
async def generate_images(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[str]:
|
||||
"""Generate one or more images from text. Returns URLs or base64-encoded images.
|
||||
|
||||
Args:
|
||||
prompt: Description of the image(s) to generate.
|
||||
settings: Execution settings for the prompt.
|
||||
kwargs: Additional arguments, check the openai images.generate documentation for the supported arguments.
|
||||
|
||||
Returns:
|
||||
list[str]: Image URLs or base64-encoded images.
|
||||
|
||||
Example:
|
||||
Generate images and save them as PNG files:
|
||||
|
||||
```python
|
||||
from semantic_kernel.connectors.ai.open_ai import AzureTextToImage
|
||||
import base64, os
|
||||
|
||||
service = AzureTextToImage(
|
||||
service_id="image1",
|
||||
deployment_name="gpt-image-1",
|
||||
endpoint="https://your-endpoint.cognitiveservices.azure.com",
|
||||
api_key="your-api-key",
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
settings = service.get_prompt_execution_settings_class()(service_id="image1")
|
||||
settings.n = 3
|
||||
images_b64 = await service.generate_images("A cute cat wearing a whimsical striped hat", settings=settings)
|
||||
```
|
||||
"""
|
||||
if not settings:
|
||||
settings = OpenAITextToImageExecutionSettings(**kwargs)
|
||||
if not isinstance(settings, OpenAITextToImageExecutionSettings):
|
||||
settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
|
||||
if prompt:
|
||||
settings.prompt = prompt
|
||||
|
||||
if not settings.prompt:
|
||||
raise ServiceInvalidRequestError("Prompt is required.")
|
||||
|
||||
if not settings.ai_model_id:
|
||||
settings.ai_model_id = self.ai_model_id
|
||||
|
||||
response = await self._send_request(settings)
|
||||
|
||||
assert isinstance(response, ImagesResponse) # nosec
|
||||
if not response.data or not isinstance(response.data, list) or len(response.data) == 0:
|
||||
raise ServiceResponseException("Failed to generate image.")
|
||||
|
||||
results: list[str] = []
|
||||
for image in response.data:
|
||||
url: str | None = getattr(image, "url", None)
|
||||
b64_json: str | None = getattr(image, "b64_json", None)
|
||||
if url:
|
||||
results.append(url)
|
||||
elif b64_json:
|
||||
results.append(b64_json)
|
||||
else:
|
||||
continue
|
||||
|
||||
if len(results) == 0:
|
||||
raise ServiceResponseException("No valid image data found in response.")
|
||||
return results
|
||||
|
||||
async def edit_image(
|
||||
self,
|
||||
prompt: str,
|
||||
image_paths: list[str] | None = None,
|
||||
image_files: list[IO[bytes]] | None = None,
|
||||
mask_path: str | None = None,
|
||||
mask_file: IO[bytes] | None = None,
|
||||
settings: PromptExecutionSettings | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[str]:
|
||||
"""Edit images using the OpenAI image edit API.
|
||||
|
||||
Args:
|
||||
prompt: Instructional prompt for image editing.
|
||||
image_paths: List of image file paths to edit.
|
||||
image_files: List of file-like objects (opened in binary mode) to edit.
|
||||
mask_path: Optional mask image file path.
|
||||
mask_file: Optional mask image file-like object (opened in binary mode).
|
||||
settings: Optional execution settings. If not provided, will be constructed from kwargs.
|
||||
kwargs: Additional API parameters.
|
||||
|
||||
Returns:
|
||||
list[str]: List of edited image URLs or base64-encoded strings.
|
||||
|
||||
Example:
|
||||
Edit images from file path and save results:
|
||||
|
||||
```python
|
||||
from semantic_kernel.connectors.ai.open_ai import AzureTextToImage
|
||||
import base64, os
|
||||
|
||||
service = AzureTextToImage(
|
||||
service_id="image1",
|
||||
deployment_name="gpt-image-1",
|
||||
endpoint="https://your-endpoint.cognitiveservices.azure.com",
|
||||
api_key="your-api-key",
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
file_paths = ["./new_images/img_1.png", "./new_images/img_2.png"]
|
||||
settings = service.get_prompt_execution_settings_class()(service_id="image1")
|
||||
settings.n = 2
|
||||
results = await service.edit_image(
|
||||
prompt="Make the cat wear a wizard hat",
|
||||
image_paths=file_paths,
|
||||
settings=settings,
|
||||
)
|
||||
```
|
||||
|
||||
Edit images from file object:
|
||||
|
||||
```python
|
||||
with open("./new_images/img_1.png", "rb") as f:
|
||||
results = await service.edit_image(
|
||||
prompt="Make the cat wear a wizard hat",
|
||||
image_files=[f],
|
||||
)
|
||||
```
|
||||
"""
|
||||
if not settings:
|
||||
settings = OpenAITextToImageExecutionSettings(**kwargs)
|
||||
if not isinstance(settings, OpenAITextToImageExecutionSettings):
|
||||
settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
|
||||
settings.prompt = prompt
|
||||
|
||||
if not settings.prompt:
|
||||
raise ServiceInvalidRequestError("Prompt is required.")
|
||||
if (image_paths is None and image_files is None) or (image_paths is not None and image_files is not None):
|
||||
raise ServiceInvalidRequestError("Provide either 'image_paths' or 'image_files', and only one.")
|
||||
|
||||
images: list[FileTypes] = []
|
||||
if image_paths is not None:
|
||||
images = [Path(p) for p in image_paths]
|
||||
elif image_files is not None:
|
||||
images = list(image_files)
|
||||
|
||||
mask: FileTypes | Omit = omit
|
||||
if mask_path is not None:
|
||||
mask = Path(mask_path)
|
||||
elif mask_file is not None:
|
||||
mask = mask_file
|
||||
|
||||
response: ImagesResponse = await self._send_image_edit_request(
|
||||
image=images,
|
||||
mask=mask,
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
if not response or not response.data or not isinstance(response.data, list):
|
||||
raise ServiceResponseException("Failed to edit image.")
|
||||
|
||||
results: list[str] = []
|
||||
for img in response.data:
|
||||
b64_json: str | None = getattr(img, "b64_json", None)
|
||||
url: str | None = getattr(img, "url", None)
|
||||
if b64_json:
|
||||
results.append(b64_json)
|
||||
elif url:
|
||||
results.append(url)
|
||||
if not results:
|
||||
raise ServiceResponseException("No valid image data found in response.")
|
||||
return results
|
||||
|
||||
def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
|
||||
"""Get the request settings class."""
|
||||
return OpenAITextToImageExecutionSettings
|
||||
Reference in New Issue
Block a user