chore: import upstream snapshot with attribution

python/semantic_kernel/connectors/ai/open_ai/__init__.py

@@ -0,0 +1,102 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import (
+    ApiKeyAuthentication,
+    AzureAISearchDataSource,
+    AzureAISearchDataSourceParameters,
+    AzureChatPromptExecutionSettings,
+    AzureCosmosDBDataSource,
+    AzureCosmosDBDataSourceParameters,
+    AzureDataSourceParameters,
+    AzureEmbeddingDependency,
+    ConnectionStringAuthentication,
+    DataSourceFieldsMapping,
+    ExtraBody,
+)
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_audio_to_text_execution_settings import (
+    OpenAIAudioToTextExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
+    OpenAIChatPromptExecutionSettings,
+    OpenAIEmbeddingPromptExecutionSettings,
+    OpenAIPromptExecutionSettings,
+    OpenAITextPromptExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
+    AzureRealtimeExecutionSettings,
+    InputAudioTranscription,
+    OpenAIRealtimeExecutionSettings,
+    TurnDetection,
+)
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_audio_execution_settings import (
+    OpenAITextToAudioExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_image_execution_settings import (
+    OpenAITextToImageExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import ListenEvents, SendEvents
+from semantic_kernel.connectors.ai.open_ai.services.azure_audio_to_text import AzureAudioToText
+from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion
+from semantic_kernel.connectors.ai.open_ai.services.azure_realtime import AzureRealtimeWebRTC, AzureRealtimeWebsocket
+from semantic_kernel.connectors.ai.open_ai.services.azure_text_completion import AzureTextCompletion
+from semantic_kernel.connectors.ai.open_ai.services.azure_text_embedding import AzureTextEmbedding
+from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_audio import AzureTextToAudio
+from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_image import AzureTextToImage
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text import OpenAIAudioToText
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import (
+    OpenAIRealtimeWebRTC,
+    OpenAIRealtimeWebsocket,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio import OpenAITextToAudio
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image import OpenAITextToImage
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+
+__all__ = [
+    "ApiKeyAuthentication",
+    "AzureAISearchDataSource",
+    "AzureAISearchDataSourceParameters",
+    "AzureAudioToText",
+    "AzureChatCompletion",
+    "AzureChatPromptExecutionSettings",
+    "AzureCosmosDBDataSource",
+    "AzureCosmosDBDataSourceParameters",
+    "AzureDataSourceParameters",
+    "AzureEmbeddingDependency",
+    "AzureOpenAISettings",
+    "AzureRealtimeExecutionSettings",
+    "AzureRealtimeWebRTC",
+    "AzureRealtimeWebsocket",
+    "AzureTextCompletion",
+    "AzureTextEmbedding",
+    "AzureTextToAudio",
+    "AzureTextToImage",
+    "ConnectionStringAuthentication",
+    "DataSourceFieldsMapping",
+    "DataSourceFieldsMapping",
+    "ExtraBody",
+    "InputAudioTranscription",
+    "ListenEvents",
+    "OpenAIAudioToText",
+    "OpenAIAudioToTextExecutionSettings",
+    "OpenAIChatCompletion",
+    "OpenAIChatPromptExecutionSettings",
+    "OpenAIEmbeddingPromptExecutionSettings",
+    "OpenAIPromptExecutionSettings",
+    "OpenAIRealtimeExecutionSettings",
+    "OpenAIRealtimeWebRTC",
+    "OpenAIRealtimeWebsocket",
+    "OpenAISettings",
+    "OpenAITextCompletion",
+    "OpenAITextEmbedding",
+    "OpenAITextPromptExecutionSettings",
+    "OpenAITextToAudio",
+    "OpenAITextToAudioExecutionSettings",
+    "OpenAITextToImage",
+    "OpenAITextToImageExecutionSettings",
+    "SendEvents",
+    "TurnDetection",
+]

python/semantic_kernel/connectors/ai/open_ai/const.py

@@ -0,0 +1,5 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import Final
+
+DEFAULT_AZURE_API_VERSION: Final[str] = "2025-08-28"

python/semantic_kernel/connectors/ai/open_ai/exceptions/content_filter_ai_exception.py

@@ -0,0 +1,90 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+from openai import BadRequestError
+
+from semantic_kernel.exceptions import ServiceContentFilterException
+
+
+class ContentFilterResultSeverity(Enum):
+    """The severity of the content filter result."""
+
+    HIGH = "high"
+    MEDIUM = "medium"
+    SAFE = "safe"
+    LOW = "low"
+
+
+@dataclass
+class ContentFilterResult:
+    """The result of a content filter check."""
+
+    filtered: bool = False
+    detected: bool = False
+    severity: ContentFilterResultSeverity = ContentFilterResultSeverity.SAFE
+
+    @classmethod
+    def from_inner_error_result(cls, inner_error_results: dict[str, Any]) -> "ContentFilterResult":
+        """Creates a ContentFilterResult from the inner error results.
+
+        Args:
+            key (str): The key to get the inner error result from.
+            inner_error_results (Dict[str, Any]): The inner error results.
+
+        Returns:
+            ContentFilterResult: The ContentFilterResult.
+        """
+        return cls(
+            filtered=inner_error_results.get("filtered", False),
+            detected=inner_error_results.get("detected", False),
+            severity=ContentFilterResultSeverity(
+                inner_error_results.get("severity", ContentFilterResultSeverity.SAFE.value)
+            ),
+        )
+
+
+class ContentFilterCodes(Enum):
+    """Content filter codes."""
+
+    RESPONSIBLE_AI_POLICY_VIOLATION = "ResponsibleAIPolicyViolation"
+
+
+@dataclass
+class ContentFilterAIException(ServiceContentFilterException):
+    """AI exception for an error from Azure OpenAI's content filter."""
+
+    # The parameter that caused the error.
+    param: str | None
+
+    # The error code specific to the content filter.
+    content_filter_code: ContentFilterCodes
+
+    # The results of the different content filter checks.
+    content_filter_result: dict[str, ContentFilterResult]
+
+    def __init__(
+        self,
+        message: str,
+        inner_exception: BadRequestError,
+    ) -> None:
+        """Initializes a new instance of the ContentFilterAIException class.
+
+        Args:
+            message (str): The error message.
+            inner_exception (Exception): The inner exception.
+        """
+        super().__init__(message)
+
+        self.param = inner_exception.param
+        if inner_exception.body is not None and isinstance(inner_exception.body, dict):
+            inner_error = inner_exception.body.get("innererror", {})
+            self.content_filter_code = ContentFilterCodes(
+                inner_error.get("code", ContentFilterCodes.RESPONSIBLE_AI_POLICY_VIOLATION.value)
+            )
+            self.content_filter_result = {
+                key: ContentFilterResult.from_inner_error_result(values)
+                for key, values in inner_error.get("content_filter_result", {}).items()
+            }

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/azure_chat_prompt_execution_settings.py

@@ -0,0 +1,171 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
+
+from pydantic import AliasGenerator, ConfigDict, Field
+from pydantic.alias_generators import to_camel, to_snake
+from pydantic.functional_validators import AfterValidator
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
+    OpenAIChatPromptExecutionSettings,
+)
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+
+if TYPE_CHECKING:
+    from semantic_kernel.connectors.azure_ai_search import AzureAISearchSettings
+
+logger = logging.getLogger(__name__)
+
+
+class AzureChatRequestBase(KernelBaseModel):
+    """Base class for Azure Chat requests."""
+
+    model_config = ConfigDict(
+        alias_generator=AliasGenerator(validation_alias=to_camel, serialization_alias=to_snake),
+        use_enum_values=True,
+        extra="allow",
+    )
+
+
+class ConnectionStringAuthentication(AzureChatRequestBase):
+    """Connection string authentication."""
+
+    type: Annotated[Literal["ConnectionString", "connection_string"], AfterValidator(to_snake)] = "connection_string"
+    connection_string: str | None = None
+
+
+class ApiKeyAuthentication(AzureChatRequestBase):
+    """API key authentication."""
+
+    type: Annotated[Literal["APIKey", "api_key"], AfterValidator(to_snake)] = "api_key"
+    key: str
+
+
+class SystemAssignedManagedIdentityAuthentication(AzureChatRequestBase):
+    """System assigned managed identity authentication."""
+
+    type: Annotated[
+        Literal["SystemAssignedManagedIdentity", "system_assigned_managed_identity"], AfterValidator(to_snake)
+    ] = "system_assigned_managed_identity"
+
+
+class UserAssignedManagedIdentityAuthentication(AzureChatRequestBase):
+    """User assigned managed identity authentication."""
+
+    type: Annotated[
+        Literal["UserAssignedManagedIdentity", "user_assigned_managed_identity"], AfterValidator(to_snake)
+    ] = "user_assigned_managed_identity"
+    managed_identity_resource_id: str | None
+
+
+class AccessTokenAuthentication(AzureChatRequestBase):
+    """Access token authentication."""
+
+    type: Annotated[Literal["AccessToken", "access_token"], AfterValidator(to_snake)] = "access_token"
+    access_token: str | None
+
+
+class AzureEmbeddingDependency(AzureChatRequestBase):
+    """Azure embedding dependency."""
+
+    type: Annotated[Literal["DeploymentName", "deployment_name"], AfterValidator(to_snake)] = "deployment_name"
+    deployment_name: str | None = None
+
+
+class DataSourceFieldsMapping(AzureChatRequestBase):
+    """Data source fields mapping."""
+
+    title_field: str | None = None
+    url_field: str | None = None
+    filepath_field: str | None = None
+    content_fields: list[str] | None = None
+    vector_fields: list[str] | None = None
+    content_fields_separator: str | None = "\n"
+
+
+class AzureDataSourceParameters(AzureChatRequestBase):
+    """Azure data source parameters."""
+
+    index_name: str
+    index_language: str | None = None
+    fields_mapping: DataSourceFieldsMapping | None = None
+    in_scope: bool | None = True
+    top_n_documents: int | None = 5
+    semantic_configuration: str | None = None
+    role_information: str | None = None
+    filter: str | None = None
+    strictness: int = 3
+    embedding_dependency: AzureEmbeddingDependency | None = None
+
+
+class AzureCosmosDBDataSourceParameters(AzureDataSourceParameters):
+    """Azure Cosmos DB data source parameters."""
+
+    authentication: ConnectionStringAuthentication | None = None
+    database_name: str | None = None
+    container_name: str | None = None
+    embedding_dependency_type: AzureEmbeddingDependency | None = None
+
+
+class AzureCosmosDBDataSource(AzureChatRequestBase):
+    """Azure Cosmos DB data source."""
+
+    type: Literal["azure_cosmos_db"] = "azure_cosmos_db"
+    parameters: AzureCosmosDBDataSourceParameters
+
+
+class AzureAISearchDataSourceParameters(AzureDataSourceParameters):
+    """Azure AI Search data source parameters."""
+
+    endpoint: str | None = None
+    query_type: Annotated[
+        Literal["simple", "semantic", "vector", "vectorSimpleHybrid", "vectorSemanticHybrid"], AfterValidator(to_snake)
+    ] = "simple"
+    authentication: (
+        ApiKeyAuthentication
+        | SystemAssignedManagedIdentityAuthentication
+        | UserAssignedManagedIdentityAuthentication
+        | AccessTokenAuthentication
+        | None
+    ) = None
+
+
+class AzureAISearchDataSource(AzureChatRequestBase):
+    """Azure AI Search data source."""
+
+    type: Literal["azure_search"] = "azure_search"
+    parameters: Annotated[dict, AzureAISearchDataSourceParameters]
+
+    @classmethod
+    def from_azure_ai_search_settings(cls, azure_ai_search_settings: "AzureAISearchSettings", **kwargs: Any):
+        """Create an instance from Azure AI Search settings."""
+        kwargs["parameters"] = {
+            "endpoint": str(azure_ai_search_settings.endpoint),
+            "index_name": azure_ai_search_settings.index_name,
+            "authentication": {
+                "key": azure_ai_search_settings.api_key.get_secret_value() if azure_ai_search_settings.api_key else None
+            },
+        }
+        return cls(**kwargs)
+
+
+DataSource = Annotated[Union[AzureAISearchDataSource, AzureCosmosDBDataSource], Field(discriminator="type")]
+
+
+class ExtraBody(KernelBaseModel):
+    """Extra body for the Azure Chat Completion endpoint."""
+
+    data_sources: list[DataSource] | None = None
+    input_language: Annotated[str | None, Field(serialization_alias="inputLanguage")] = None
+    output_language: Annotated[str | None, Field(serialization_alias="outputLanguage")] = None
+
+    def __getitem__(self, item):
+        """Get an item from the ExtraBody."""
+        return getattr(self, item)
+
+
+class AzureChatPromptExecutionSettings(OpenAIChatPromptExecutionSettings):
+    """Specific settings for the Azure OpenAI Chat Completion endpoint."""
+
+    extra_body: dict[str, Any] | ExtraBody | None = None  # type: ignore[assignment]

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_audio_to_text_execution_settings.py

@@ -0,0 +1,33 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import Any
+
+from pydantic import Field
+
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIAudioToTextExecutionSettings(PromptExecutionSettings):
+    """Request settings for OpenAI audio to text services."""
+
+    ai_model_id: str | None = Field(default=None, serialization_alias="model")
+    filename: str | None = Field(
+        default=None,
+        description="Do not set this manually. It is set by the service based on the audio content.",
+    )
+    language: str | None = None
+    prompt: str | None = None
+    response_format: str | None = None
+    temperature: float | None = None
+
+    def prepare_settings_dict(self, **kwargs) -> dict[str, Any]:
+        """Prepare the settings dictionary for the OpenAI API."""
+        settings_dict = super().prepare_settings_dict(**kwargs)
+
+        # Remove the file name since it will be open as a file object
+        settings_dict.pop("filename", None)
+
+        return settings_dict

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py

@@ -0,0 +1,161 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIPromptExecutionSettings(PromptExecutionSettings):
+    """Common request settings for (Azure) OpenAI services."""
+
+    ai_model_id: Annotated[str | None, Field(serialization_alias="model")] = None
+    frequency_penalty: Annotated[float | None, Field(ge=-2.0, le=2.0)] = None
+    logit_bias: dict[str | int, float] | None = None
+    max_tokens: Annotated[int | None, Field(gt=0)] = None
+    number_of_responses: Annotated[int | None, Field(ge=1, le=128, serialization_alias="n")] = None
+    presence_penalty: Annotated[float | None, Field(ge=-2.0, le=2.0)] = None
+    seed: int | None = None
+    stop: str | list[str] | None = None
+    stream: bool = False
+    temperature: Annotated[float | None, Field(ge=0.0, le=2.0)] = None
+    top_p: Annotated[float | None, Field(ge=0.0, le=1.0)] = None
+    user: str | None = None
+    store: bool | None = None
+    metadata: dict[str, str] | None = None
+
+
+class OpenAITextPromptExecutionSettings(OpenAIPromptExecutionSettings):
+    """Specific settings for the completions endpoint."""
+
+    prompt: Annotated[
+        str | None, Field(description="Do not set this manually. It is set by the service based on the text content.")
+    ] = None
+    best_of: Annotated[int | None, Field(ge=1)] = None
+    echo: bool = False
+    logprobs: Annotated[int | None, Field(ge=0, le=5)] = None
+    suffix: str | None = None
+
+    @model_validator(mode="after")
+    def check_best_of_and_n(self) -> "OpenAITextPromptExecutionSettings":
+        """Check that the best_of parameter is not greater than the number_of_responses parameter."""
+        best_of = self.best_of or self.extension_data.get("best_of")
+        number_of_responses = self.number_of_responses or self.extension_data.get("number_of_responses")
+
+        if best_of is not None and number_of_responses is not None and best_of < number_of_responses:
+            raise ServiceInvalidExecutionSettingsError(
+                "When used with number_of_responses, best_of controls the number of candidate completions and n specifies how many to return, therefore best_of must be greater than number_of_responses."  # noqa: E501
+            )
+
+        return self
+
+
+class OpenAIChatPromptExecutionSettings(OpenAIPromptExecutionSettings):
+    """Specific settings for the Chat Completion endpoint."""
+
+    response_format: (
+        dict[Literal["type"], Literal["text", "json_object"]] | dict[str, Any] | type[BaseModel] | type | None
+    ) = None
+    function_call: str | None = None
+    functions: list[dict[str, Any]] | None = None
+    messages: Annotated[
+        list[dict[str, Any]] | None, Field(description="Do not set this manually. It is set by the service.")
+    ] = None
+    parallel_tool_calls: bool | None = None
+    tools: Annotated[
+        list[dict[str, Any]] | None,
+        Field(
+            description="Do not set this manually. It is set by the service based "
+            "on the function choice configuration.",
+        ),
+    ] = None
+    tool_choice: Annotated[
+        str | None,
+        Field(
+            description="Do not set this manually. It is set by the service based "
+            "on the function choice configuration.",
+        ),
+    ] = None
+    structured_json_response: Annotated[
+        bool, Field(description="Do not set this manually. It is set by the service.")
+    ] = False
+    stream_options: Annotated[
+        dict[str, Any] | None,
+        Field(description="Additional options to pass when streaming is used. Do not set this manually."),
+    ] = None
+    max_completion_tokens: Annotated[
+        int | None,
+        Field(
+            gt=0,
+            description="A maximum limit on total tokens for completion, including both output and reasoning tokens.",
+        ),
+    ] = None
+    reasoning_effort: Annotated[
+        Literal["low", "medium", "high"] | None,
+        Field(
+            description="Adjusts reasoning effort (low/medium/high). Lower values reduce response time and token usage."
+        ),
+    ] = None
+    extra_body: dict[str, Any] | None = None
+
+    @field_validator("functions", "function_call", mode="after")
+    @classmethod
+    def validate_function_call(cls, v: str | list[dict[str, Any]] | None = None):
+        """Validate the function_call and functions parameters."""
+        if v is not None:
+            logger.warning(
+                "The function_call and functions parameters are deprecated. Please use the tool_choice and tools parameters instead."  # noqa: E501
+            )
+        return v
+
+    @model_validator(mode="before")
+    def validate_response_format_and_set_flag(cls, values: Any) -> Any:
+        """Validate the response_format and set structured_json_response accordingly."""
+        if not isinstance(values, dict):
+            return values
+        response_format = values.get("response_format", None)
+
+        if response_format is None:
+            return values
+
+        if isinstance(response_format, dict):
+            if response_format.get("type") == "json_object":
+                return values
+            if response_format.get("type") == "json_schema":
+                json_schema = response_format.get("json_schema")
+                if isinstance(json_schema, dict):
+                    values["structured_json_response"] = True
+                    return values
+                raise ServiceInvalidExecutionSettingsError(
+                    "If response_format has type 'json_schema', 'json_schema' must be a valid dictionary."
+                )
+        if isinstance(response_format, type):
+            if issubclass(response_format, BaseModel):
+                values["structured_json_response"] = True
+            else:
+                values["structured_json_response"] = True
+        else:
+            raise ServiceInvalidExecutionSettingsError(
+                "response_format must be a dictionary, a subclass of BaseModel, a Python class/type, or None"
+            )
+
+        return values
+
+
+class OpenAIEmbeddingPromptExecutionSettings(PromptExecutionSettings):
+    """Specific settings for the text embedding endpoint."""
+
+    input: str | list[str] | list[int] | list[list[int]] | None = None
+    ai_model_id: Annotated[str | None, Field(serialization_alias="model")] = None
+    encoding_format: Literal["float", "base64"] | None = None
+    user: str | None = None
+    extra_headers: dict | None = None
+    extra_query: dict | None = None
+    extra_body: dict | None = None
+    timeout: float | None = None
+    dimensions: Annotated[int | None, Field(gt=0, le=3072)] = None

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py

@@ -0,0 +1,129 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping, Sequence
+from typing import Annotated, Any, Literal
+
+from pydantic import Field
+
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+
+
+class InputAudioTranscription(KernelBaseModel):
+    """Input audio transcription settings.
+
+    Args:
+        model: The model to use for transcription, should be one of the following:
+            - whisper-1
+            - gpt-4o-transcribe
+            - gpt-4o-mini-transcribe
+        language: The language of the audio, should be in ISO-639-1 format, like 'en'.
+        prompt: An optional text to guide the model's style or continue a previous audio segment.
+            The prompt should match the audio language.
+    """
+
+    model: Literal["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"] | None = None
+    language: str | None = None
+    prompt: str | None = None
+
+
+class TurnDetection(KernelBaseModel):
+    """Turn detection settings.
+
+    Args:
+        type: The type of turn detection, server_vad or semantic_vad.
+        create_response: Whether to create a response for each detected turn.
+        eagerness: The eagerness of the voice activity detection, can be low, medium, high, or auto,
+            used only for semantic_vad.
+        interrupt_response: Whether to interrupt the response for each detected turn.
+        prefix_padding_ms: The padding before the detected voice activity, in milliseconds.
+        silence_duration_ms: The duration of silence to detect the end of a turn, in milliseconds.
+        threshold: The threshold for voice activity detection, should be between 0 and 1, only for server_vad.
+
+    """
+
+    type: Literal["server_vad", "semantic_vad"] = "server_vad"
+    create_response: bool | None = None
+    eagerness: Literal["low", "medium", "high", "auto"] | None = None
+    interrupt_response: bool | None = None
+    prefix_padding_ms: Annotated[int | None, Field(ge=0)] = None
+    silence_duration_ms: Annotated[int | None, Field(ge=0)] = None
+    threshold: Annotated[float | None, Field(ge=0.0, le=1.0)] = None
+
+
+class OpenAIRealtimeExecutionSettings(PromptExecutionSettings):
+    """Request settings for OpenAI realtime services."""
+
+    output_modalities: Sequence[Literal["audio", "text"]] | None = None
+    ai_model_id: Annotated[str | None, Field(None, serialization_alias="model")] = None
+    instructions: str | None = None
+    voice: str | None = None
+    input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None
+    output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None
+    input_audio_transcription: InputAudioTranscription | Mapping[str, str] | None = None
+    turn_detection: TurnDetection | Mapping[str, str] | None = None
+    tools: Annotated[
+        list[dict[str, Any]] | None,
+        Field(
+            description="Do not set this manually. It is set by the service based "
+            "on the function choice configuration.",
+        ),
+    ] = None
+    tool_choice: Annotated[
+        str | None,
+        Field(
+            description="Do not set this manually. It is set by the service based "
+            "on the function choice configuration.",
+        ),
+    ] = None
+    max_output_tokens: Annotated[int | Literal["inf"] | None, Field(gt=0)] = None
+    input_audio_noise_reduction: dict[Literal["type"], Literal["near_field", "far_field"]] | None = None
+
+    def prepare_settings_dict(self, **kwargs) -> dict[str, Any]:
+        """Prepare the settings as a dictionary for sending to the AI service.
+
+        For realtime settings, we need to properly structure the audio configuration
+        to match the OpenAI API expectations where voice and turn_detection are nested
+        under the audio field.
+        """
+        # Get the base settings dict (excludes service_id, extension_data, etc.)
+        settings_dict = super().prepare_settings_dict(**kwargs)
+
+        # Build the audio configuration object
+        audio_config: dict[str, Any] = {}
+
+        # Handle voice (goes in audio.output.voice)
+        if "voice" in settings_dict:
+            audio_config.setdefault("output", {})["voice"] = settings_dict.pop("voice")
+
+        # Handle turn_detection (goes in audio.input.turn_detection)
+        if "turn_detection" in settings_dict:
+            audio_config.setdefault("input", {})["turn_detection"] = settings_dict.pop("turn_detection")
+
+        # Handle input audio format
+        if "input_audio_format" in settings_dict:
+            audio_config.setdefault("input", {})["format"] = settings_dict.pop("input_audio_format")
+
+        # Handle output audio format
+        if "output_audio_format" in settings_dict:
+            audio_config.setdefault("output", {})["format"] = settings_dict.pop("output_audio_format")
+
+        # Handle input audio transcription
+        if "input_audio_transcription" in settings_dict:
+            audio_config.setdefault("input", {})["transcription"] = settings_dict.pop("input_audio_transcription")
+
+        # Handle input audio noise reduction
+        if "input_audio_noise_reduction" in settings_dict:
+            audio_config.setdefault("input", {})["noise_reduction"] = settings_dict.pop("input_audio_noise_reduction")
+
+        # Add the audio config if it has any content
+        if audio_config:
+            settings_dict["audio"] = audio_config
+
+        return settings_dict
+
+
+class AzureRealtimeExecutionSettings(OpenAIRealtimeExecutionSettings):
+    """Request settings for Azure OpenAI realtime services."""
+
+    pass

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_audio_execution_settings.py

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import Annotated, Literal
+
+from pydantic import Field
+
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAITextToAudioExecutionSettings(PromptExecutionSettings):
+    """Request settings for OpenAI text to audio services."""
+
+    ai_model_id: str | None = Field(None, serialization_alias="model")
+    input: str | None = Field(
+        None, description="Do not set this manually. It is set by the service based on the text content."
+    )
+    voice: Literal["alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"] = "alloy"
+    response_format: Literal["mp3", "opus", "aac", "flac", "wav", "pcm"] | None = None
+    speed: Annotated[float | None, Field(ge=0.25, le=4.0)] = None

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_text_to_image_execution_settings.py

@@ -0,0 +1,84 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import Any, Literal
+
+from pydantic import Field, model_validator
+
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInvalidExecutionSettingsError
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+
+logger = logging.getLogger(__name__)
+
+
+VALID_IMAGE_SIZES = [
+    (256, 256),
+    (512, 512),
+    (1024, 1024),
+    (1792, 1024),
+    (1024, 1792),
+]
+
+
+class ImageSize(KernelBaseModel):
+    """Image size."""
+
+    width: int
+    height: int
+
+    def __str__(self) -> str:
+        """Return the string representation of the image size."""
+        return f"{self.width}x{self.height}"
+
+
+class OpenAITextToImageExecutionSettings(PromptExecutionSettings):
+    """Request settings for OpenAI text to image services."""
+
+    prompt: str | None = None
+    ai_model_id: str | None = Field(default=None, serialization_alias="model")
+    size: ImageSize | None = None
+    quality: Literal["high", "medium", "low"] | None = None
+    output_compression: int | None = None
+    background: Literal["transparent", "opaque", "auto"] | None = None
+    n: int | None = Field(default=1, ge=1, le=10)
+    moderation: Literal["auto", "low"] | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def get_size(cls, data: dict[str, Any]) -> dict[str, Any]:
+        """Check that the requested image size is valid."""
+        if isinstance(data, dict):
+            if "size" not in data and "width" in data and "height" in data:
+                data["size"] = ImageSize(width=data["width"], height=data["height"])
+            elif "extension_data" in data:
+                extension_data = data["extension_data"]
+                if (
+                    isinstance(extension_data, dict)
+                    and "size" not in extension_data
+                    and "width" in extension_data
+                    and "height" in extension_data
+                ):
+                    data["extension_data"]["size"] = ImageSize(
+                        width=extension_data["width"], height=extension_data["height"]
+                    )
+        return data
+
+    @model_validator(mode="after")
+    def check_size(self) -> "OpenAITextToImageExecutionSettings":
+        """Check that the requested image size is valid."""
+        size = self.size or self.extension_data.get("size")
+
+        if size is not None and (size.width, size.height) not in VALID_IMAGE_SIZES:
+            raise ServiceInvalidExecutionSettingsError(f"Invalid image size: {size.width}x{size.height}.")
+
+        return self
+
+    def prepare_settings_dict(self, **kwargs) -> dict[str, Any]:
+        """Prepare the settings dictionary for the OpenAI API."""
+        settings_dict = super().prepare_settings_dict(**kwargs)
+
+        if self.size is not None:
+            settings_dict["size"] = str(self.size)
+
+        return settings_dict

python/semantic_kernel/connectors/ai/open_ai/services/_open_ai_realtime.py

@@ -0,0 +1,995 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+import base64
+import contextlib
+import json
+import logging
+import sys
+from collections.abc import AsyncGenerator, Callable, Coroutine
+from enum import Enum
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
+
+import numpy as np
+from aiohttp import ClientSession
+from aiortc import (
+    MediaStreamTrack,
+    RTCConfiguration,
+    RTCDataChannel,
+    RTCIceServer,
+    RTCPeerConnection,
+    RTCSessionDescription,
+)
+from av.audio.frame import AudioFrame
+from numpy import ndarray
+from openai._models import construct_type_unchecked
+from openai.resources.realtime.realtime import AsyncRealtimeConnection
+from openai.types.realtime import (
+    ConversationItemCreateEvent,
+    ConversationItemDeleteEvent,
+    ConversationItemTruncateEvent,
+    InputAudioBufferAppendEvent,
+    InputAudioBufferClearEvent,
+    InputAudioBufferCommitEvent,
+    RealtimeClientEvent,
+    RealtimeConversationItemFunctionCall,
+    RealtimeConversationItemFunctionCallOutput,
+    RealtimeConversationItemUserMessage,
+    RealtimeResponseCreateParams,
+    RealtimeServerEvent,
+    ResponseCancelEvent,
+    ResponseCreateEvent,
+    ResponseFunctionCallArgumentsDoneEvent,
+    SessionUpdateEvent,
+)
+from pydantic import Field, PrivateAttr
+
+from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
+from semantic_kernel.connectors.ai.function_calling_utils import prepare_settings_for_function_calling
+from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
+    OpenAIRealtimeExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase
+from semantic_kernel.const import USER_AGENT
+from semantic_kernel.contents.audio_content import AudioContent
+from semantic_kernel.contents.chat_history import ChatHistory
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.function_call_content import FunctionCallContent
+from semantic_kernel.contents.function_result_content import FunctionResultContent
+from semantic_kernel.contents.realtime_events import (
+    RealtimeAudioEvent,
+    RealtimeEvent,
+    RealtimeEvents,
+    RealtimeFunctionCallEvent,
+    RealtimeFunctionResultEvent,
+    RealtimeTextEvent,
+)
+from semantic_kernel.contents.streaming_text_content import StreamingTextContent
+from semantic_kernel.contents.text_content import TextContent
+from semantic_kernel.exceptions import ContentException
+from semantic_kernel.kernel import Kernel
+from semantic_kernel.utils.feature_stage_decorator import experimental
+from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT, prepend_semantic_kernel_to_user_agent
+
+if TYPE_CHECKING:
+    from aiortc.mediastreams import MediaStreamTrack
+
+    from semantic_kernel.connectors.ai.function_choice_behavior import (
+        FunctionCallChoiceConfiguration,
+        FunctionChoiceType,
+    )
+    from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+    from semantic_kernel.contents.chat_history import ChatHistory
+    from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
+
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+logger: logging.Logger = logging.getLogger("semantic_kernel.connectors.ai.open_ai.realtime")
+
+
+# region utils
+
+
+def update_settings_from_function_call_configuration(
+    function_choice_configuration: "FunctionCallChoiceConfiguration",
+    settings: "PromptExecutionSettings",
+    type: "FunctionChoiceType",
+) -> None:
+    """Update the settings from a FunctionChoiceConfiguration."""
+    if (
+        function_choice_configuration.available_functions
+        and hasattr(settings, "tool_choice")
+        and hasattr(settings, "tools")
+    ):
+        settings.tool_choice = type  # type: ignore
+        settings.tools = [  # type: ignore
+            kernel_function_metadata_to_function_call_format(f)
+            for f in function_choice_configuration.available_functions
+        ]
+
+
+def kernel_function_metadata_to_function_call_format(
+    metadata: "KernelFunctionMetadata",
+) -> dict[str, Any]:
+    """Convert the kernel function metadata to function calling format.
+
+    Function calling in the realtime API, uses a slightly different format than the chat completion API.
+    See https://platform.openai.com/docs/api-reference/realtime-sessions/create#realtime-sessions-create-tools
+    for more details.
+
+    TLDR: there is no "function" key, and the function details are at the same level as "type".
+    """
+    return {
+        "type": "function",
+        "name": metadata.fully_qualified_name,
+        "description": metadata.description or "",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                param.name: param.schema_data for param in metadata.parameters if param.include_in_function_choices
+            },
+            "required": [p.name for p in metadata.parameters if p.is_required and p.include_in_function_choices],
+        },
+    }
+
+
+# region constants
+
+
+@experimental
+class SendEvents(str, Enum):
+    """Events that can be sent."""
+
+    SESSION_UPDATE = "session.update"
+    INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append"
+    INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit"
+    INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear"
+    CONVERSATION_ITEM_CREATE = "conversation.item.create"
+    CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate"
+    CONVERSATION_ITEM_DELETE = "conversation.item.delete"
+    RESPONSE_CREATE = "response.create"
+    RESPONSE_CANCEL = "response.cancel"
+
+
+def _create_openai_realtime_client_event(event_type: SendEvents | str, **kwargs: Any) -> RealtimeClientEvent:
+    """Create an OpenAI Realtime client event from a event type and kwargs."""
+    if isinstance(event_type, str):
+        event_type = SendEvents(event_type)
+    match event_type:
+        case SendEvents.SESSION_UPDATE:
+            if "session" not in kwargs:
+                raise ContentException("Session is required for SessionUpdateEvent")
+            session_dict = kwargs.pop("session")
+            # Create proper RealtimeSessionCreateRequest with required type field for SDK validation
+            # The OpenAI SDK will handle the proper serialization for the API
+            from openai.types.realtime import RealtimeSessionCreateRequest
+
+            session_request = RealtimeSessionCreateRequest(type="realtime", **session_dict)
+            return SessionUpdateEvent(
+                type=event_type.value,
+                session=session_request,
+                **kwargs,
+            )
+        case SendEvents.INPUT_AUDIO_BUFFER_APPEND:
+            if "audio" not in kwargs:
+                raise ContentException("Audio is required for InputAudioBufferAppendEvent")
+            return InputAudioBufferAppendEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+        case SendEvents.INPUT_AUDIO_BUFFER_COMMIT:
+            return InputAudioBufferCommitEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+        case SendEvents.INPUT_AUDIO_BUFFER_CLEAR:
+            return InputAudioBufferClearEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+        case SendEvents.CONVERSATION_ITEM_CREATE:
+            if "item" not in kwargs:
+                raise ContentException("Item is required for ConversationItemCreateEvent")
+            kwargs["type"] = event_type.value
+            return ConversationItemCreateEvent(**kwargs)
+        case SendEvents.CONVERSATION_ITEM_TRUNCATE:
+            if "content_index" not in kwargs:
+                kwargs["content_index"] = 0
+            return ConversationItemTruncateEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+        case SendEvents.CONVERSATION_ITEM_DELETE:
+            if "item_id" not in kwargs:
+                raise ContentException("Item ID is required for ConversationItemDeleteEvent")
+            return ConversationItemDeleteEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+        case SendEvents.RESPONSE_CREATE:
+            if "response" in kwargs:
+                response: RealtimeResponseCreateParams | None = RealtimeResponseCreateParams.model_validate(
+                    kwargs.pop("response")
+                )
+            else:
+                response = None
+            return ResponseCreateEvent(
+                type=event_type.value,
+                response=response,
+                **kwargs,
+            )
+        case SendEvents.RESPONSE_CANCEL:
+            return ResponseCancelEvent(
+                type=event_type.value,
+                **kwargs,
+            )
+
+
+@experimental
+class ListenEvents(str, Enum):
+    """Events that can be listened to."""
+
+    ERROR = "error"
+    SESSION_CREATED = "session.created"
+    SESSION_UPDATED = "session.updated"
+    CONVERSATION_CREATED = "conversation.created"
+    INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed"
+    INPUT_AUDIO_BUFFER_CLEARED = "input_audio_buffer.cleared"
+    INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started"
+    INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped"
+    CONVERSATION_ITEM_CREATED = "conversation.item.created"
+    CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED = "conversation.item.input_audio_transcription.completed"
+    CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED = "conversation.item.input_audio_transcription.failed"
+    CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated"
+    CONVERSATION_ITEM_DELETED = "conversation.item.deleted"
+    RESPONSE_CREATED = "response.created"
+    RESPONSE_DONE = "response.done"  # contains usage info -> log
+    RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added"
+    RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done"
+    RESPONSE_CONTENT_PART_ADDED = "response.content_part.added"
+    RESPONSE_CONTENT_PART_DONE = "response.content_part.done"
+    RESPONSE_TEXT_DELTA = "response.output_text.delta"
+    RESPONSE_TEXT_DONE = "response.output_text.done"
+    RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.output_audio_transcript.delta"
+    RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.output_audio_transcript.done"
+    RESPONSE_AUDIO_DELTA = "response.output_audio.delta"
+    RESPONSE_AUDIO_DONE = "response.output_audio.done"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
+    RATE_LIMITS_UPDATED = "rate_limits.updated"
+
+
+# region Base
+
+
+@experimental
+class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase):
+    """OpenAI Realtime service."""
+
+    SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True
+
+    _current_settings: PromptExecutionSettings | None = PrivateAttr(default=None)
+    _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict)
+
+    def model_post_init(self, __context: Any) -> None:
+        """Post init hook."""
+        super().model_post_init(__context)
+        if self.model_extra:
+            if "kernel" in self.model_extra:
+                self._kernel = self.model_extra["kernel"]
+            if "plugins" in self.model_extra:
+                self._add_plugin_to_kernel(self.model_extra["plugins"])
+            if "settings" in self.model_extra:
+                self._current_settings = self.model_extra["settings"]
+            if "chat_history" in self.model_extra:
+                self._chat_history = self.model_extra["chat_history"]
+
+    async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[RealtimeEvents, None]:
+        """Handle all events but audio delta.
+
+        Audio delta has to be handled by the implementation of the protocol as some
+        protocols have different ways of handling audio.
+
+        We put all event in the output buffer, but after the interpreted one.
+        so when dealing with them, make sure to check the type of the event, since they
+        might be of different types.
+        """
+        match event.type:
+            case (
+                ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value
+                | "response.audio_transcript.delta"
+                | ListenEvents.RESPONSE_TEXT_DELTA.value
+                | "response.text.delta"
+            ):
+                yield RealtimeTextEvent(
+                    service_type=event.type,
+                    service_event=event,
+                    text=StreamingTextContent(
+                        inner_content=event,
+                        text=event.delta,  # type: ignore
+                        choice_index=0,
+                    ),
+                )
+            case (
+                ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE.value
+                | "response.audio_transcript.done"
+                | ListenEvents.RESPONSE_TEXT_DONE.value
+                | "response.text.done"
+            ):
+                # Don't yield RealtimeTextEvent here — the deltas already streamed all
+                # the text.  Emitting the full text again would cause duplicate output
+                # for any consumer that prints every RealtimeTextEvent.
+                yield RealtimeEvent(service_type=event.type, service_event=event)
+            case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED.value:
+                if event.item.type == "function_call" and event.item.call_id and event.item.name:  # type: ignore
+                    self._call_id_to_function_map[event.item.call_id] = event.item.name  # type: ignore
+                yield RealtimeEvent(service_type=event.type, service_event=event)
+            case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA.value:
+                yield RealtimeFunctionCallEvent(
+                    service_type=event.type,
+                    service_event=event,
+                    function_call=FunctionCallContent(
+                        id=event.item_id,  # type: ignore
+                        name=self._call_id_to_function_map[event.call_id],  # type: ignore
+                        arguments=event.delta,  # type: ignore
+                        index=event.output_index,  # type: ignore
+                        metadata={"call_id": event.call_id},  # type: ignore
+                        inner_content=event,
+                    ),
+                )
+            case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE.value:
+                async for parsed_event in self._parse_function_call_arguments_done(event):  # type: ignore
+                    if parsed_event:
+                        yield parsed_event
+            case ListenEvents.ERROR.value:
+                # In GA API, event.error is a dict instead of an object
+                error_info = event.error if isinstance(event.error, dict) else event.error.model_dump()  # type: ignore
+                logger.error("Error received: %s", error_info)  # type: ignore
+                yield RealtimeEvent(service_type=event.type, service_event=event)
+            case ListenEvents.SESSION_CREATED.value | ListenEvents.SESSION_UPDATED.value:
+                logger.info("Session created or updated, session: %s", event.session.model_dump_json())  # type: ignore
+                yield RealtimeEvent(service_type=event.type, service_event=event)
+            case _:
+                logger.debug(f"Received event: {event}")
+                yield RealtimeEvent(service_type=event.type, service_event=event)
+
+    @override
+    async def update_session(
+        self,
+        chat_history: ChatHistory | None = None,
+        settings: PromptExecutionSettings | None = None,
+        create_response: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        """Update the session in the service.
+
+        Args:
+            chat_history: Chat history.
+            settings: Prompt execution settings, if kernel is linked to the service or passed as
+                Kwargs, it will be used to update the settings for function calling.
+            create_response: Create a response, get the model to start responding, default is False.
+            kwargs: Additional arguments, if 'kernel' or 'plugins' is passed, it will be used to update the
+                settings for function calling, others will be ignored.
+
+        """
+        if kwargs:
+            if self._create_kwargs:
+                kwargs = {**self._create_kwargs, **kwargs}
+        else:
+            kwargs = self._create_kwargs or {}
+        if settings:
+            self._current_settings = settings
+        if "kernel" in kwargs:
+            self._kernel = kwargs["kernel"]
+        if "plugins" in kwargs:
+            self._add_plugin_to_kernel(kwargs["plugins"])
+
+        if self._current_settings:
+            if self._kernel:
+                self._current_settings = prepare_settings_for_function_calling(
+                    self._current_settings,
+                    self.get_prompt_execution_settings_class(),
+                    self._update_function_choice_settings_callback(),
+                    kernel=self._kernel,
+                )
+            await self.send(
+                RealtimeEvent(
+                    service_type=SendEvents.SESSION_UPDATE,
+                    service_event={"settings": self._current_settings},
+                )
+            )
+
+        if chat_history and len(chat_history) > 0:
+            for msg in chat_history.messages:
+                for item in msg.items:
+                    match item:
+                        case TextContent():
+                            await self.send(
+                                RealtimeTextEvent(service_type=SendEvents.CONVERSATION_ITEM_CREATE, text=item)
+                            )
+                        case FunctionCallContent():
+                            await self.send(
+                                RealtimeFunctionCallEvent(
+                                    service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_call=item
+                                )
+                            )
+                        case FunctionResultContent():
+                            await self.send(
+                                RealtimeFunctionResultEvent(
+                                    service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_result=item
+                                )
+                            )
+                        case _:
+                            logger.error("Unsupported item type: %s", item)
+
+        if create_response or kwargs.get("create_response", False) is True:
+            await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
+
+    def _add_plugin_to_kernel(self, plugins: list[object] | dict[str, object]) -> None:
+        if not self._kernel:
+            self._kernel = Kernel()
+        if isinstance(plugins, list):
+            plugins = {p.__class__.__name__: p for p in plugins}
+        self._kernel.add_plugins(plugins)
+
+    async def _parse_function_call_arguments_done(
+        self,
+        event: ResponseFunctionCallArgumentsDoneEvent,
+    ) -> AsyncGenerator[RealtimeEvents | None]:
+        """Handle response function call done.
+
+        This always yields at least 1 event, either a RealtimeEvent or a RealtimeFunctionResultEvent with the raw event.
+
+        It then also yields any function results both back to the service, through `send` and to the developer.
+
+        """
+        # Step 1: check if function calling enabled:
+        if not self._kernel or (
+            self._current_settings
+            and self._current_settings.function_choice_behavior
+            and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions
+        ):
+            yield RealtimeEvent(service_type=event.type, service_event=event)
+            return
+        # Step 2: check if there is a function that can be found.
+        try:
+            plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1)
+        except ValueError:
+            logger.error("Function call needs to have a plugin name and function name")
+            yield RealtimeEvent(service_type=event.type, service_event=event)
+            return
+
+        # Step 3: Parse into the function call content, and yield that.
+        item = FunctionCallContent(
+            id=event.item_id,
+            plugin_name=plugin_name,
+            function_name=function_name,
+            arguments=event.arguments,
+            index=event.output_index,
+            metadata={"call_id": event.call_id},
+        )
+        yield RealtimeFunctionCallEvent(
+            service_type=ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, function_call=item, service_event=event
+        )
+
+        # Step 4: Invoke the function call
+        # Fail closed: only invoke when a function choice behavior is available so the allowlist can be enforced.
+        # Without it, kernel.invoke_function_call would skip allowlist validation and execute any named function.
+        function_behavior = self._current_settings.function_choice_behavior if self._current_settings else None
+        if function_behavior is None:
+            logger.warning(
+                "Skipping function call '%s-%s' because no function choice behavior is configured; "
+                "allowlist validation cannot be enforced.",
+                plugin_name,
+                function_name,
+            )
+            created_output = FunctionResultContent.from_function_call_content_and_result(
+                function_call_content=item,
+                result="Function call was not invoked because function choice behavior is not configured.",
+            )
+            result = RealtimeFunctionResultEvent(
+                service_type=SendEvents.CONVERSATION_ITEM_CREATE,
+                function_result=created_output,
+            )
+            await self.send(result)
+            await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
+            yield result
+            return
+        chat_history = ChatHistory()
+        await self._kernel.invoke_function_call(item, chat_history, function_behavior=function_behavior)
+        created_output: FunctionResultContent = chat_history.messages[-1].items[0]  # type: ignore
+        # Step 5: Create the function result event
+        result = RealtimeFunctionResultEvent(
+            service_type=SendEvents.CONVERSATION_ITEM_CREATE,
+            function_result=created_output,
+        )
+        # Step 6: send the result to the service and call `create response`
+        await self.send(result)
+        await self.send(RealtimeEvent(service_type=SendEvents.RESPONSE_CREATE))
+        # Step 7: yield the function result back to the developer as well
+        yield result
+
+    async def _send(self, event: RealtimeClientEvent) -> None:
+        """Send an event to the service."""
+        raise NotImplementedError
+
+    @override
+    async def send(self, event: RealtimeEvents, **kwargs: Any) -> None:
+        match event:
+            case RealtimeAudioEvent():
+                await self._send(
+                    _create_openai_realtime_client_event(
+                        event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, audio=event.audio.data_string
+                    )
+                )
+            case RealtimeTextEvent():
+                await self._send(
+                    _create_openai_realtime_client_event(
+                        event_type=SendEvents.CONVERSATION_ITEM_CREATE,
+                        item=RealtimeConversationItemUserMessage(
+                            type="message",
+                            content=[
+                                {
+                                    "type": "input_text",
+                                    "text": event.text.text,
+                                }
+                            ],
+                            role="user",
+                        ),
+                    )
+                )
+            case RealtimeFunctionCallEvent():
+                await self._send(
+                    _create_openai_realtime_client_event(
+                        event_type=SendEvents.CONVERSATION_ITEM_CREATE,
+                        item=RealtimeConversationItemFunctionCall(
+                            type="function_call",
+                            name=event.function_call.name or event.function_call.function_name,
+                            arguments=""
+                            if not event.function_call.arguments
+                            else event.function_call.arguments
+                            if isinstance(event.function_call.arguments, str)
+                            else json.dumps(event.function_call.arguments),
+                            call_id=event.function_call.metadata.get("call_id"),
+                        ),
+                    )
+                )
+            case RealtimeFunctionResultEvent():
+                await self._send(
+                    _create_openai_realtime_client_event(
+                        event_type=SendEvents.CONVERSATION_ITEM_CREATE,
+                        item=RealtimeConversationItemFunctionCallOutput(
+                            type="function_call_output",
+                            output=event.function_result.result,
+                            call_id=event.function_result.metadata.get("call_id"),
+                        ),
+                    )
+                )
+            case _:
+                data = event.service_event
+                match event.service_type:
+                    case SendEvents.SESSION_UPDATE:
+                        if not data:
+                            logger.error("Event data is empty")
+                            return
+                        settings = data.get("settings", None)
+                        if not settings:
+                            logger.error("Event data does not contain 'settings'")
+                            return
+                        try:
+                            settings = self.get_prompt_execution_settings_from_settings(settings)
+                        except Exception as e:
+                            logger.error(
+                                f"Failed to properly create settings from passed settings: {settings}, error: {e}"
+                            )
+                            return
+                        assert isinstance(settings, self.get_prompt_execution_settings_class())  # nosec
+                        if not settings.ai_model_id:  # type: ignore
+                            settings.ai_model_id = self.ai_model_id  # type: ignore
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type,
+                                session=settings.prepare_settings_dict(),
+                            )
+                        )
+                    case SendEvents.INPUT_AUDIO_BUFFER_APPEND:
+                        if not data or "audio" not in data:
+                            logger.error("Event data does not contain 'audio'")
+                            return
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type,
+                                audio=data["audio"],
+                            )
+                        )
+                    case SendEvents.INPUT_AUDIO_BUFFER_COMMIT:
+                        await self._send(_create_openai_realtime_client_event(event_type=event.service_type))
+                    case SendEvents.INPUT_AUDIO_BUFFER_CLEAR:
+                        await self._send(_create_openai_realtime_client_event(event_type=event.service_type))
+                    case SendEvents.CONVERSATION_ITEM_CREATE:
+                        if not data or "item" not in data:
+                            logger.error("Event data does not contain 'item'")
+                            return
+                        content = data["item"]
+                        contents = content.items if isinstance(content, ChatMessageContent) else [content]
+                        for item in contents:
+                            match item:
+                                case TextContent():
+                                    await self._send(
+                                        _create_openai_realtime_client_event(
+                                            event_type=event.service_type,
+                                            item=RealtimeConversationItemUserMessage(
+                                                type="message",
+                                                content=[
+                                                    {
+                                                        "type": "input_text",
+                                                        "text": item.text,
+                                                    }
+                                                ],
+                                                role="user",
+                                            ),
+                                        )
+                                    )
+                                case FunctionCallContent():
+                                    await self._send(
+                                        _create_openai_realtime_client_event(
+                                            event_type=event.service_type,
+                                            item=RealtimeConversationItemFunctionCall(
+                                                type="function_call",
+                                                name=item.name or item.function_name,
+                                                arguments=""
+                                                if not item.arguments
+                                                else item.arguments
+                                                if isinstance(item.arguments, str)
+                                                else json.dumps(item.arguments),
+                                                call_id=item.metadata.get("call_id"),
+                                            ),
+                                        )
+                                    )
+
+                                case FunctionResultContent():
+                                    await self._send(
+                                        _create_openai_realtime_client_event(
+                                            event_type=event.service_type,
+                                            item=RealtimeConversationItemFunctionCallOutput(
+                                                type="function_call_output",
+                                                output=item.result,
+                                                call_id=item.metadata.get("call_id"),
+                                            ),
+                                        )
+                                    )
+                    case SendEvents.CONVERSATION_ITEM_TRUNCATE:
+                        if not data or "item_id" not in data:
+                            logger.error("Event data does not contain 'item_id'")
+                            return
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type,
+                                item_id=data["item_id"],
+                                content_index=0,
+                                audio_end_ms=data.get("audio_end_ms", 0),
+                            )
+                        )
+                    case SendEvents.CONVERSATION_ITEM_DELETE:
+                        if not data or "item_id" not in data:
+                            logger.error("Event data does not contain 'item_id'")
+                            return
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type,
+                                item_id=data["item_id"],
+                            )
+                        )
+                    case SendEvents.RESPONSE_CREATE:
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type, event_id=data.get("event_id", None) if data else None
+                            )
+                        )
+                    case SendEvents.RESPONSE_CANCEL:
+                        await self._send(
+                            _create_openai_realtime_client_event(
+                                event_type=event.service_type,
+                                response_id=data.get("response_id", None) if data else None,
+                            )
+                        )
+
+    @override
+    def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
+        return OpenAIRealtimeExecutionSettings
+
+    @override
+    def _update_function_choice_settings_callback(
+        self,
+    ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]:
+        return update_settings_from_function_call_configuration
+
+
+# region WebRTC
+@experimental
+class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase):
+    """OpenAI WebRTC Realtime service."""
+
+    peer_connection: RTCPeerConnection | None = None
+    data_channel: RTCDataChannel | None = None
+    audio_track: MediaStreamTrack | None = None
+    _receive_buffer: asyncio.Queue[RealtimeEvents] = PrivateAttr(default_factory=asyncio.Queue)
+
+    @override
+    async def receive(
+        self,
+        audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None,
+        **kwargs: Any,
+    ) -> AsyncGenerator[RealtimeEvents, None]:
+        if audio_output_callback:
+            self.audio_output_callback = audio_output_callback
+        while True:
+            yield await self._receive_buffer.get()
+
+    async def _send(self, event: RealtimeClientEvent) -> None:
+        if not self.data_channel:
+            logger.error("Data channel not initialized")
+            return
+        while self.data_channel.readyState != "open":
+            await asyncio.sleep(0.1)
+        try:
+            # Handle session update specially to exclude type field for WebRTC
+            if hasattr(event, "type") and event.type == "session.update":
+                event_dict = event.model_dump(exclude_none=True)
+                # Remove fields that aren't allowed in session updates for WebRTC compatibility
+                # Audio configuration should be set during session creation, not updates
+                session_dict = event_dict.get("session")
+                if session_dict and isinstance(session_dict, dict):
+                    # Only keep fields that are allowed in session updates
+                    # Note: output_modalities is not allowed in WebRTC session updates
+                    allowed_fields = {
+                        "type",
+                        "instructions",
+                        "model",
+                        "max_output_tokens",
+                        "tools",
+                        "tool_choice",
+                        "prompt",
+                        "tracing",
+                        "truncation",
+                    }
+                    event_dict["session"] = {k: v for k, v in session_dict.items() if k in allowed_fields}
+
+                self.data_channel.send(json.dumps(event_dict))
+            else:
+                self.data_channel.send(event.model_dump_json(exclude_none=True))
+        except Exception as e:
+            logger.error(f"Failed to send event {event} with error: {e!s}")
+
+    @override
+    async def create_session(
+        self,
+        chat_history: "ChatHistory | None" = None,
+        settings: "PromptExecutionSettings | None" = None,
+        **kwargs: Any,
+    ) -> None:
+        """Create a session in the service."""
+        if not self.audio_track:
+            raise Exception("Audio track not initialized")
+        self.peer_connection = RTCPeerConnection(
+            configuration=RTCConfiguration(iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")])
+        )
+
+        # track is the audio track being returned from the service
+        self.peer_connection.add_listener("track", self._on_track)
+
+        # data channel is used to send and receive messages
+        self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json")
+        self.data_channel.add_listener("message", self._on_data)
+
+        # this is the incoming audio, which sends audio to the service
+        self.peer_connection.addTransceiver(self.audio_track)
+
+        offer = await self.peer_connection.createOffer()
+        await self.peer_connection.setLocalDescription(offer)
+
+        try:
+            ephemeral_token = await self._get_ephemeral_token()
+            headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"}
+            headers = prepend_semantic_kernel_to_user_agent(headers)
+
+            async with (
+                ClientSession() as session,
+                session.post(self._get_webrtc_url(), headers=headers, data=offer.sdp) as response,
+            ):
+                if response.status not in [200, 201]:
+                    error_text = await response.text()
+                    raise Exception(f"OpenAI WebRTC error: {error_text}")
+
+                sdp_answer = await response.text()
+                answer = RTCSessionDescription(sdp=sdp_answer, type="answer")
+                await self.peer_connection.setRemoteDescription(answer)
+                logger.info("Connected to OpenAI WebRTC")
+
+        except Exception as e:
+            logger.error(f"Failed to connect to OpenAI: {e!s}")
+            raise
+
+        await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
+
+    @override
+    async def close_session(self) -> None:
+        """Close the session in the service."""
+        if self.peer_connection:
+            with contextlib.suppress(asyncio.CancelledError):
+                await self.peer_connection.close()
+        self.peer_connection = None
+        if self.data_channel:
+            with contextlib.suppress(asyncio.CancelledError):
+                self.data_channel.close()
+        self.data_channel = None
+
+    async def _on_track(self, track: "MediaStreamTrack") -> None:
+        logger.debug(f"Received {track.kind} track from remote")
+        if track.kind != "audio":
+            return
+        while True:
+            try:
+                # This is a MediaStreamTrack, so the type is AudioFrame
+                # this might need to be updated if video becomes part of this
+                frame: AudioFrame = await track.recv()  # type: ignore
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error getting audio frame: {e!s}")
+                break
+
+            try:
+                if self.audio_output_callback:
+                    await self.audio_output_callback(frame.to_ndarray())
+
+            except Exception as e:
+                logger.error(f"Error playing remote audio frame: {e!s}")
+            try:
+                await self._receive_buffer.put(
+                    RealtimeAudioEvent(
+                        audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame),
+                        service_event=frame,
+                        service_type=ListenEvents.RESPONSE_AUDIO_DELTA,
+                    ),
+                )
+            except Exception as e:
+                logger.error(f"Error processing remote audio frame: {e!s}")
+            await asyncio.sleep(0.01)
+
+    async def _on_data(self, data: str) -> None:
+        """This method is called whenever a data channel message is received.
+
+        The data is parsed into a RealtimeServerEvent (by OpenAI code) and then processed.
+        Audio data is not send through this channel, use _on_track for that.
+        """
+        try:
+            event = cast(
+                RealtimeServerEvent,
+                construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)),
+            )
+        except Exception as e:
+            logger.error(f"Failed to parse event {data} with error: {e!s}")
+            return
+        async for parsed_event in self._parse_event(event):
+            await self._receive_buffer.put(parsed_event)
+
+    async def _get_ephemeral_token(self) -> str:
+        """Get an ephemeral token from OpenAI.
+
+        GA endpoint: POST /v1/realtime/client_secrets
+        Request body: {"session": {"type": "realtime", "model": "<model>"}}
+        Response: {"value": "<token>", "expires_at": ..., "session": {...}}
+        """
+        data = {
+            "session": {
+                "type": "realtime",
+                "model": self.ai_model_id,
+            }
+        }
+        headers, url = self._get_ephemeral_token_headers_and_url()
+        headers = prepend_semantic_kernel_to_user_agent(headers)
+        try:
+            async with (
+                ClientSession() as session,
+                session.post(url, headers=headers, json=data) as response,
+            ):
+                if response.status not in [200, 201]:
+                    error_text = await response.text()
+                    raise Exception(f"Failed to get ephemeral token: {error_text}")
+
+                result = await response.json()
+                return result["value"]
+
+        except Exception as e:
+            logger.error(f"Failed to get ephemeral token: {e!s}")
+            raise
+
+    def _get_ephemeral_token_headers_and_url(self) -> tuple[dict[str, str], str]:
+        """Get the headers and URL for the ephemeral token."""
+        return {
+            "Authorization": f"Bearer {self.client.api_key}",
+            "Content-Type": "application/json",
+        }, f"{self.client.realtime._client.base_url}/realtime/client_secrets"
+
+    def _get_webrtc_url(self) -> str:
+        """Get the WebRTC URL.
+
+        GA endpoint: POST /v1/realtime/calls?model=<model>
+        """
+        return f"{self.client.realtime._client.base_url}/realtime/calls?model={self.ai_model_id}"
+
+
+# region Websocket
+
+
+@experimental
+class OpenAIRealtimeWebsocketBase(OpenAIRealtimeBase):
+    """OpenAI Realtime service."""
+
+    protocol: ClassVar[Literal["websocket"]] = "websocket"  # type: ignore
+    connection: AsyncRealtimeConnection | None = None
+    connected: asyncio.Event = Field(default_factory=asyncio.Event)
+
+    @override
+    async def receive(
+        self,
+        audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None,
+        **kwargs: Any,
+    ) -> AsyncGenerator[RealtimeEvents, None]:
+        if audio_output_callback:
+            self.audio_output_callback = audio_output_callback
+        await self.connected.wait()
+        if not self.connection:
+            raise ValueError("Connection is not established.")
+
+        async for event in self.connection:
+            if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value:
+                if self.audio_output_callback:
+                    await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16))
+                yield RealtimeAudioEvent(
+                    audio=AudioContent(data=event.delta, data_format="base64", inner_content=event),
+                    service_type=event.type,
+                    service_event=event,
+                )
+                continue
+            async for realtime_event in self._parse_event(event):
+                yield realtime_event
+
+    async def _send(self, event: RealtimeClientEvent) -> None:
+        await self.connected.wait()
+        if not self.connection:
+            raise ValueError("Connection is not established.")
+        try:
+            await self.connection.send(event)
+        except Exception as e:
+            logger.error(f"Error sending response: {e!s}")
+
+    @override
+    async def create_session(
+        self,
+        chat_history: "ChatHistory | None" = None,
+        settings: "PromptExecutionSettings | None" = None,
+        **kwargs: Any,
+    ) -> None:
+        """Create a session in the service."""
+        self.connection = await self.client.realtime.connect(
+            model=self.ai_model_id, extra_headers={USER_AGENT: SEMANTIC_KERNEL_USER_AGENT}
+        ).enter()
+        self.connected.set()
+        await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
+
+    @override
+    async def close_session(self) -> None:
+        """Close the session in the service."""
+        if self.connected.is_set() and self.connection:
+            await self.connection.close()
+            self.connection = None
+            self.connected.clear()

python/semantic_kernel/connectors/ai/open_ai/services/azure_audio_to_text.py

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text_base import OpenAIAudioToTextBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="AzureAudioToText")
+
+
+class AzureAudioToText(AzureOpenAIConfigBase, OpenAIAudioToTextBase):
+    """Azure audio to text service."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureAudioToText service.
+
+        Args:
+            service_id: The service ID. (Optional)
+            api_key: The optional api key. If provided, will override the value in the
+                    env vars or .env file.
+            deployment_name: The optional deployment. If provided, will override the value
+                (audio_to_text_deployment_name) in the env vars or .env file.
+            endpoint: The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url: The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version: The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token: The Azure AD token for authentication. (Optional)
+            ad_token_provider: Azure AD Token provider. (Optional)
+            token_endpoint: The Azure AD token endpoint. (Optional)
+            default_headers: The default headers mapping of string keys to
+                    string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+            credential: The credential to use for authentication. (Optional)
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+                api_key=api_key,
+                audio_to_text_deployment_name=deployment_name,
+                endpoint=endpoint,
+                base_url=base_url,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as exc:
+            raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
+        if not azure_openai_settings.audio_to_text_deployment_name:
+            raise ServiceInitializationError("The Azure OpenAI audio to text deployment name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.audio_to_text_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.AUDIO_TO_TEXT,
+            client=async_client,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: deployment_name, endpoint, api_key
+                and optionally: api_version, ad_auth
+        """
+        return cls(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/azure_chat_completion.py

@@ -0,0 +1,221 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import json
+import logging
+from collections.abc import Mapping
+from copy import deepcopy
+from typing import Any, TypeVar
+from uuid import uuid4
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from openai.types.chat.chat_completion import ChatCompletion, Choice
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import (
+    AzureChatPromptExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base import OpenAIChatCompletionBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.function_call_content import FunctionCallContent
+from semantic_kernel.contents.function_result_content import FunctionResultContent
+from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
+from semantic_kernel.contents.text_content import TextContent
+from semantic_kernel.contents.utils.finish_reason import FinishReason
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+TChatMessageContent = TypeVar("TChatMessageContent", ChatMessageContent, StreamingChatMessageContent)
+
+
+class AzureChatCompletion(AzureOpenAIConfigBase, OpenAIChatCompletionBase, OpenAITextCompletionBase):
+    """Azure Chat completion class."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        instruction_role: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureChatCompletion service.
+
+        Args:
+            service_id (str | None): The service ID for the Azure deployment. (Optional)
+            api_key  (str | None): The optional api key. If provided, will override the value in the
+                env vars or .env file.
+            deployment_name  (str | None): The optional deployment. If provided, will override the value
+                (chat_deployment_name) in the env vars or .env file.
+            endpoint (str | None): The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url (str | None): The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version (str | None): The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token (str | None): The Azure Active Directory token. (Optional)
+            ad_token_provider (AsyncAzureADTokenProvider): The Azure Active Directory token provider. (Optional)
+            token_endpoint (str | None): The token endpoint to request an Azure token. (Optional)
+            default_headers (Mapping[str, str]): The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client (AsyncAzureOpenAI | None): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback to using env vars.
+            env_file_encoding (str | None): The encoding of the environment settings file, defaults to 'utf-8'.
+            instruction_role (str | None): The role to use for 'instruction' messages, for example, summarization
+                prompts could use `developer` or `system`. (Optional)
+            credential (TokenCredential): The credential to use for authentication.
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                api_key=api_key,
+                base_url=base_url,
+                endpoint=endpoint,
+                chat_deployment_name=deployment_name,
+                api_version=api_version,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as exc:
+            raise ServiceInitializationError(f"Failed to validate settings: {exc}") from exc
+
+        if not azure_openai_settings.chat_deployment_name:
+            raise ServiceInitializationError("chat_deployment_name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.chat_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.CHAT,
+            client=async_client,
+            instruction_role=instruction_role,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls, settings: dict[str, Any]) -> "AzureChatCompletion":
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: service_id, and optionally:
+                    ad_auth, ad_token_provider, default_headers
+        """
+        return AzureChatCompletion(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )
+
+    def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
+        """Create a request settings object."""
+        return AzureChatPromptExecutionSettings
+
+    def _create_chat_message_content(
+        self, response: ChatCompletion, choice: Choice, response_metadata: dict[str, Any]
+    ) -> ChatMessageContent:
+        """Create an Azure chat message content object from a choice."""
+        content = super()._create_chat_message_content(response, choice, response_metadata)
+        return self._add_tool_message_to_chat_message_content(content, choice)
+
+    def _create_streaming_chat_message_content(
+        self,
+        chunk: ChatCompletionChunk,
+        choice: ChunkChoice,
+        chunk_metadata: dict[str, Any],
+        function_invoke_attempt: int = 0,
+    ) -> "StreamingChatMessageContent":
+        """Create an Azure streaming chat message content object from a choice."""
+        content = super()._create_streaming_chat_message_content(chunk, choice, chunk_metadata, function_invoke_attempt)
+        assert isinstance(content, StreamingChatMessageContent) and isinstance(choice, ChunkChoice)  # nosec
+        return self._add_tool_message_to_chat_message_content(content, choice)
+
+    def _add_tool_message_to_chat_message_content(
+        self,
+        content: TChatMessageContent,
+        choice: Choice | ChunkChoice,
+    ) -> TChatMessageContent:
+        if tool_message := self._get_tool_message_from_chat_choice(choice=choice):
+            if not isinstance(tool_message, dict):
+                # try to json, to ensure it is a dictionary
+                try:
+                    tool_message = json.loads(tool_message)
+                except json.JSONDecodeError:
+                    logger.warning("Tool message is not a dictionary, ignore context.")
+                    return content
+            function_call = FunctionCallContent(
+                id=str(uuid4()),
+                name="Azure-OnYourData",
+                arguments=json.dumps({"query": tool_message.get("intent", [])}),
+            )
+            result = FunctionResultContent.from_function_call_content_and_result(
+                result=tool_message["citations"], function_call_content=function_call
+            )
+            content.items.insert(0, function_call)
+            content.items.insert(1, result)
+        return content
+
+    def _get_tool_message_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[str, Any] | None:
+        """Get the tool message from a choice."""
+        content = choice.message if isinstance(choice, Choice) else choice.delta
+        # When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
+        if content and content.model_extra is not None:
+            return content.model_extra.get("context", None)
+        # openai allows extra content, so model_extra will be a dict, but we need to check anyway, but no way to test.
+        return None  # pragma: no cover
+
+    @staticmethod
+    def split_message(message: "ChatMessageContent") -> list["ChatMessageContent"]:
+        """Split an Azure On Your Data response into separate ChatMessageContents.
+
+        If the message does not have three contents, and those three are one each of:
+        FunctionCallContent, FunctionResultContent, and TextContent,
+        it will not return three messages, potentially only one or two.
+
+        The order of the returned messages is as expected by OpenAI.
+        """
+        if len(message.items) != 3:
+            return [message]
+        messages = {"tool_call": deepcopy(message), "tool_result": deepcopy(message), "assistant": deepcopy(message)}
+        for key, msg in messages.items():
+            if key == "tool_call":
+                msg.items = [item for item in msg.items if isinstance(item, FunctionCallContent)]
+                msg.finish_reason = FinishReason.FUNCTION_CALL
+            if key == "tool_result":
+                msg.items = [item for item in msg.items if isinstance(item, FunctionResultContent)]
+            if key == "assistant":
+                msg.items = [item for item in msg.items if isinstance(item, TextContent)]
+        return [messages["tool_call"], messages["tool_result"], messages["assistant"]]

python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py

@@ -0,0 +1,151 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Awaitable, Callable, Mapping
+from copy import copy
+from typing import Any
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from pydantic import ConfigDict, validate_call
+from pydantic_core import Url
+
+from semantic_kernel.connectors.ai.open_ai.const import DEFAULT_AZURE_API_VERSION
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler, OpenAIModelTypes
+from semantic_kernel.const import USER_AGENT
+from semantic_kernel.exceptions import ServiceInitializationError
+from semantic_kernel.kernel_pydantic import HttpsUrl
+from semantic_kernel.utils.authentication.entra_id_authentication import get_entra_auth_token
+from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class AzureOpenAIConfigBase(OpenAIHandler):
+    """Internal class for configuring a connection to an Azure OpenAI service."""
+
+    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
+    def __init__(
+        self,
+        deployment_name: str,
+        ai_model_type: OpenAIModelTypes,
+        endpoint: HttpsUrl | None = None,
+        base_url: Url | None = None,
+        api_version: str = DEFAULT_AZURE_API_VERSION,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: Callable[[], str | Awaitable[str]] | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        client: AsyncAzureOpenAI | None = None,
+        instruction_role: str | None = None,
+        credential: TokenCredential | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Internal class for configuring a connection to an Azure OpenAI service.
+
+        The `validate_call` decorator is used with a configuration that allows arbitrary types.
+        This is necessary for types like `HttpsUrl` and `OpenAIModelTypes`.
+
+        Args:
+            deployment_name (str): Name of the deployment.
+            ai_model_type (OpenAIModelTypes): The type of OpenAI model to deploy.
+            endpoint (HttpsUrl): The specific endpoint URL for the deployment. (Optional)
+            base_url (Url): The base URL for Azure services. (Optional)
+            api_version (str): Azure API version. Defaults to the defined DEFAULT_AZURE_API_VERSION.
+            service_id (str): Service ID for the deployment. (Optional)
+            api_key (str): API key for Azure services. (Optional)
+            ad_token (str): Azure AD token for authentication. (Optional)
+            ad_token_provider (Callable[[], Union[str, Awaitable[str]]]): A callable
+                or coroutine function providing Azure AD tokens. (Optional)
+            token_endpoint (str): Azure AD token endpoint use to get the token. (Optional)
+            default_headers (Union[Mapping[str, str], None]): Default headers for HTTP requests. (Optional)
+            client (AsyncAzureOpenAI): An existing client to use. (Optional)
+            instruction_role (str | None): The role to use for 'instruction' messages, for example, summarization
+                prompts could use `developer` or `system`. (Optional)
+            credential: The credential to use for authentication. (Optional)
+            kwargs: Additional keyword arguments.
+
+        """
+        # Merge APP_INFO into the headers if it exists
+        merged_headers = dict(copy(default_headers)) if default_headers else {}
+        if APP_INFO:
+            merged_headers.update(APP_INFO)
+            merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers)
+
+        if not client:
+            # If the client is None, the api_key is none, the ad_token is none, and the ad_token_provider is none,
+            # then we will attempt to get the ad_token using the default endpoint specified in the Azure OpenAI
+            # settings.
+            if not api_key and not ad_token_provider and not ad_token and token_endpoint and credential:
+                ad_token = get_entra_auth_token(credential, token_endpoint)
+
+            if not api_key and not ad_token and not ad_token_provider and not credential:
+                raise ServiceInitializationError(
+                    "Please provide either api_key, ad_token, ad_token_provider, credential or a client."
+                )
+
+            if not endpoint and not base_url:
+                raise ServiceInitializationError("Please provide an endpoint or a base_url")
+
+            args: dict[str, Any] = {
+                "default_headers": merged_headers,
+            }
+            if api_version:
+                args["api_version"] = api_version
+            if ad_token:
+                args["azure_ad_token"] = ad_token
+            if ad_token_provider:
+                args["azure_ad_token_provider"] = ad_token_provider
+            if api_key:
+                args["api_key"] = api_key
+            if base_url:
+                args["base_url"] = str(base_url)
+            if endpoint and not base_url:
+                args["azure_endpoint"] = str(endpoint)
+            # TODO (eavanvalkenburg): Remove the check on model type when the package fixes: https://github.com/openai/openai-python/issues/2120
+            if deployment_name and ai_model_type != OpenAIModelTypes.REALTIME:
+                args["azure_deployment"] = deployment_name
+
+            if "websocket_base_url" in kwargs:
+                args["websocket_base_url"] = kwargs.pop("websocket_base_url")
+
+            client = AsyncAzureOpenAI(**args)
+        args = {
+            "ai_model_id": deployment_name,
+            "client": client,
+            "ai_model_type": ai_model_type,
+        }
+        if service_id:
+            args["service_id"] = service_id
+        if instruction_role:
+            args["instruction_role"] = instruction_role
+        super().__init__(**args, **kwargs)
+
+    def to_dict(self) -> dict[str, str]:
+        """Convert the configuration to a dictionary."""
+        client_settings = {
+            "base_url": str(self.client.base_url),
+            "api_version": self.client._custom_query["api-version"],
+            "api_key": self.client.api_key,
+            "ad_token": getattr(self.client, "_azure_ad_token", None),
+            "ad_token_provider": getattr(self.client, "_azure_ad_token_provider", None),
+            "default_headers": {k: v for k, v in self.client.default_headers.items() if k != USER_AGENT},
+        }
+        base = self.model_dump(
+            exclude={
+                "prompt_tokens",
+                "completion_tokens",
+                "total_tokens",
+                "api_type",
+                "org_id",
+                "ai_model_type",
+                "service_id",
+                "client",
+            },
+            by_alias=True,
+            exclude_none=True,
+        )
+        base.update(client_settings)
+        return base

python/semantic_kernel/connectors/ai/open_ai/services/azure_realtime.py

@@ -0,0 +1,387 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+import sys
+import warnings
+from collections.abc import Callable, Coroutine, Mapping
+from typing import TYPE_CHECKING, Any
+
+from aiohttp import ClientSession
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from openai.resources.realtime.realtime import AsyncRealtimeConnection
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
+    AzureRealtimeExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import (
+    OpenAIRealtimeWebRTCBase,
+    OpenAIRealtimeWebsocketBase,
+)
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.const import USER_AGENT
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.utils.feature_stage_decorator import experimental
+from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT, prepend_semantic_kernel_to_user_agent
+
+if TYPE_CHECKING:
+    from aiortc.mediastreams import MediaStreamTrack
+    from numpy import ndarray
+
+    from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+    from semantic_kernel.contents.chat_history import ChatHistory
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+logger = logging.getLogger("semantic_kernel.connectors.ai.open_ai.realtime")
+
+
+@experimental
+class AzureRealtimeWebsocket(OpenAIRealtimeWebsocketBase, AzureOpenAIConfigBase):
+    """Azure OpenAI Realtime service using WebSocket protocol."""
+
+    def __init__(
+        self,
+        audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        websocket_base_url: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        credential: TokenCredential | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize an AzureRealtimeWebsocket service.
+
+        Args:
+            audio_output_callback: The audio output callback, optional.
+                This should be a coroutine, that takes a ndarray with audio as input.
+                The goal of this function is to allow you to play the audio with the
+                least amount of latency possible, because it is called first before further processing.
+                It can also be set in the `receive` method.
+                Even when passed, the audio content will still be
+                added to the receiving queue.
+            service_id: The service ID for the Azure deployment. (Optional)
+            api_key: The optional api key. If provided, will override the value in the
+                env vars or .env file.
+            deployment_name: The optional deployment. If provided, will override the value
+                (chat_deployment_name) in the env vars or .env file.
+            endpoint: The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url: The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version: The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token: The Azure Active Directory token. (Optional)
+            ad_token_provider: The Azure Active Directory token provider. (Optional)
+            token_endpoint: The token endpoint to request an Azure token. (Optional)
+            websocket_base_url: The base URL for the WebSocket connection. (Optional)
+                If not provided, the default URL will be used.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+            credential: The credential to use for authentication. (Optional)
+            kwargs: Additional arguments.
+                This can include:
+                kernel (Kernel): the kernel to use for function calls
+                plugins (list[object] or dict[str, object]): the plugins to use for function calls
+                settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
+                chat_history (ChatHistory): the chat history to use for the session
+                Otherwise they can also be passed to the context manager.
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                api_key=api_key,
+                base_url=base_url,
+                endpoint=endpoint,
+                realtime_deployment_name=deployment_name,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not azure_openai_settings.realtime_deployment_name:
+            raise ServiceInitializationError("The OpenAI realtime model ID is required.")
+        super().__init__(
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            audio_output_callback=audio_output_callback,
+            deployment_name=azure_openai_settings.realtime_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            ai_model_type=OpenAIModelTypes.REALTIME,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+            websocket_base_url=websocket_base_url,
+            credential=credential,
+            **kwargs,
+        )
+
+    @override
+    def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
+        return AzureRealtimeExecutionSettings
+
+    @override
+    async def create_session(
+        self,
+        chat_history: "ChatHistory | None" = None,
+        settings: "PromptExecutionSettings | None" = None,
+        **kwargs: Any,
+    ) -> None:
+        """Create a session in the service.
+
+        The Azure GA Realtime endpoint (/openai/v1/realtime) does not accept
+        the api-version query parameter. The openai SDK always adds it, so we
+        bypass the SDK's _configure_realtime and build the connection directly.
+        """
+        from websockets.asyncio.client import connect as ws_connect
+
+        # Build the GA WebSocket URL: wss://<resource>.openai.azure.com/openai/v1/realtime?model=<deployment-name>
+        # Note: GA uses ?model= (not ?deployment= which was preview)
+        # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-websockets
+        endpoint = str(self.client._base_url).rstrip("/")  # type: ignore[attr-defined]
+        if "/openai" in endpoint:
+            endpoint = endpoint[: endpoint.index("/openai")]
+        url = f"wss://{endpoint.split('://')[-1]}/openai/v1/realtime?model={self.ai_model_id}"
+
+        # Build auth headers
+        auth_headers: dict[str, str] = {}
+        if self.client.api_key and self.client.api_key != "<missing API key>":
+            auth_headers["api-key"] = self.client.api_key
+        else:
+            token = await self.client._get_azure_ad_token()  # type: ignore[attr-defined]
+            if token:
+                auth_headers["Authorization"] = f"Bearer {token}"
+
+        ws = await ws_connect(
+            url,
+            additional_headers={
+                **auth_headers,
+                USER_AGENT: SEMANTIC_KERNEL_USER_AGENT,
+            },
+        )
+
+        self.connection = AsyncRealtimeConnection(ws)
+        self.connected.set()
+        await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
+
+
+@experimental
+class AzureRealtimeWebRTC(OpenAIRealtimeWebRTCBase, AzureOpenAIConfigBase):
+    """Azure OpenAI Realtime service using WebRTC protocol."""
+
+    def __init__(
+        self,
+        audio_track: "MediaStreamTrack",
+        region: str | None = None,
+        audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        credential: TokenCredential | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize an AzureRealtimeWebRTC service.
+
+        Args:
+            audio_track: The audio track to use for the service, only used by WebRTC.
+                It can be any class that implements the AudioStreamTrack interface.
+            region: Deprecated. No longer needed for GA Realtime API.
+                Previously required for the preview WebRTC endpoint.
+            audio_output_callback: The audio output callback, optional.
+                This should be a coroutine, that takes a ndarray with audio as input.
+                The goal of this function is to allow you to play the audio with the
+                least amount of latency possible, because it is called first before further processing.
+                It can also be set in the `receive` method.
+                Even when passed, the audio content will still be
+                added to the receiving queue.
+            service_id: The service ID for the Azure deployment. (Optional)
+            api_key: The optional api key. If provided, will override the value in the
+                env vars or .env file.
+            deployment_name: The optional deployment. If provided, will override the value
+                (chat_deployment_name) in the env vars or .env file.
+            endpoint: The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url: The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version: The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token: The Azure Active Directory token. (Optional)
+            ad_token_provider: The Azure Active Directory token provider. (Optional)
+            token_endpoint: The token endpoint to request an Azure token. (Optional)
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+            credential: The credential to use for authentication. (Optional)
+            kwargs: Additional arguments.
+                This can include:
+                kernel (Kernel): the kernel to use for function calls
+                plugins (list[object] or dict[str, object]): the plugins to use for function calls
+                settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
+                chat_history (ChatHistory): the chat history to use for the session
+                Otherwise they can also be passed to the context manager.
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                api_key=api_key,
+                base_url=base_url,
+                endpoint=endpoint,
+                realtime_deployment_name=deployment_name,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not azure_openai_settings.realtime_deployment_name:
+            raise ServiceInitializationError("The OpenAI realtime model ID is required.")
+        if region is not None:
+            warnings.warn(
+                "The 'region' parameter is deprecated and no longer needed for the GA Realtime API. "
+                "The WebRTC endpoint is now derived from the resource endpoint.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        if audio_track:
+            kwargs["audio_track"] = audio_track
+        super().__init__(
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            audio_output_callback=audio_output_callback,
+            deployment_name=azure_openai_settings.realtime_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            ai_model_type=OpenAIModelTypes.REALTIME,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+            credential=credential,
+            **kwargs,
+        )
+
+    @override
+    def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
+        return AzureRealtimeExecutionSettings
+
+    @override
+    def _get_ephemeral_token_headers_and_url(self) -> tuple[dict[str, str], str]:
+        """Get the headers and URL for the ephemeral token.
+
+        Uses the GA endpoint format: POST /openai/v1/realtime/client_secrets
+        See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
+        """
+        endpoint = str(self.client._base_url).rstrip("/")  # type: ignore[attr-defined]
+        # Strip any trailing path segments to get the base Azure resource URL
+        # base_url typically looks like https://<resource>.openai.azure.com/openai/...
+        # We need: https://<resource>.openai.azure.com/openai/v1/realtime/client_secrets
+        if "/openai" in endpoint:
+            endpoint = endpoint[: endpoint.index("/openai")]
+        url = f"{endpoint}/openai/v1/realtime/client_secrets"
+
+        if self.client.api_key and self.client.api_key != "<missing API key>":
+            return (
+                {
+                    "api-key": self.client.api_key,
+                    "Content-Type": "application/json",
+                },
+                url,
+            )
+        if self.client._azure_ad_token is not None:  # type: ignore[attr-defined]
+            return (
+                {
+                    "Authorization": f"Bearer {self.client._azure_ad_token}",  # type: ignore[attr-defined]
+                    "Content-Type": "application/json",
+                },
+                url,
+            )
+        raise ServiceInitializationError("No API key or Azure AD token available for ephemeral token request.")
+
+    @override
+    async def _get_ephemeral_token(self) -> str:
+        """Get an ephemeral token from Azure OpenAI.
+
+        Azure GA requires a nested session object:
+            {"session": {"type": "realtime", "model": "<deployment>"}}
+        And returns the token directly as {"value": "..."} rather than
+        OpenAI's {"client_secret": {"value": "..."}}.
+        See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
+        """
+        data = {
+            "session": {
+                "type": "realtime",
+                "model": self.ai_model_id,
+            }
+        }
+        headers, url = self._get_ephemeral_token_headers_and_url()
+        headers = prepend_semantic_kernel_to_user_agent(headers)
+        try:
+            async with (
+                ClientSession() as session,
+                session.post(url, headers=headers, json=data) as response,
+            ):
+                if response.status not in [200, 201]:
+                    error_text = await response.text()
+                    raise Exception(f"Failed to get ephemeral token: {error_text}")
+
+                result = await response.json()
+                # Azure GA format returns {"value": "token"} directly
+                return result["value"]
+
+        except Exception as e:
+            logger.error(f"Failed to get ephemeral token: {e!s}")
+            raise
+
+    @override
+    def _get_webrtc_url(self) -> str:
+        """Get the WebRTC URL.
+
+        Uses the GA endpoint format: /openai/v1/realtime/calls
+        See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/realtime-audio-webrtc
+        """
+        endpoint = str(self.client._base_url).rstrip("/")  # type: ignore[attr-defined]
+        if "/openai" in endpoint:
+            endpoint = endpoint[: endpoint.index("/openai")]
+        return f"{endpoint}/openai/v1/realtime/calls"

python/semantic_kernel/connectors/ai/open_ai/services/azure_text_completion.py

@@ -0,0 +1,127 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+import sys
+from collections.abc import Mapping
+from typing import Any
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+if sys.version_info >= (3, 13):
+    from warnings import deprecated
+else:
+    from typing_extensions import deprecated
+
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@deprecated(
+    "The AzureTextCompletion class is deprecated and will be removed after 01/01/2026. "
+    "There won't be a replacement because all text completion models on Azure OpenAI "
+    "have retired. Please migrate to chat completion models before the deprecation date."
+)
+class AzureTextCompletion(AzureOpenAIConfigBase, OpenAITextCompletionBase):
+    """Azure Text Completion class."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureTextCompletion service.
+
+        Args:
+            service_id: The service ID for the Azure deployment. (Optional)
+            api_key (str | None): The optional api key. If provided, will override the value in the
+                env vars or .env file.
+            deployment_name  (str | None): The optional deployment. If provided, will override the value
+                (text_deployment_name) in the env vars or .env file.
+            endpoint (str | None): The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url (str | None): The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version (str | None): The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token: The Azure Active Directory token. (Optional)
+            ad_token_provider: The Azure Active Directory token provider. (Optional)
+            token_endpoint: The Azure Active Directory token endpoint. (Optional)
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client (Optional[AsyncAzureOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            credential (TokenCredential): The credential to use for authentication. (Optional)
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                env_file_path=env_file_path,
+                text_deployment_name=deployment_name,
+                endpoint=endpoint,
+                base_url=base_url,
+                api_key=api_key,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError(f"Invalid settings: {ex}") from ex
+        if not azure_openai_settings.text_deployment_name:
+            raise ServiceInitializationError("The Azure Text deployment name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.text_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.TEXT,
+            client=async_client,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls, settings: dict[str, Any]) -> "AzureTextCompletion":
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: deployment_name, endpoint, api_key
+                and optionally: api_version, ad_auth
+        """
+        return AzureTextCompletion(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/azure_text_embedding.py

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding_base import OpenAITextEmbeddingBase
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.utils.feature_stage_decorator import experimental
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+@experimental
+class AzureTextEmbedding(AzureOpenAIConfigBase, OpenAITextEmbeddingBase):
+    """Azure Text Embedding class."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureTextEmbedding service.
+
+        service_id: The service ID. (Optional)
+        api_key: The optional api key. If provided, will override the value in the
+                env vars or .env file.
+        deployment_name: The optional deployment. If provided, will override the value
+            (text_deployment_name) in the env vars or .env file.
+        endpoint: The optional deployment endpoint. If provided will override the value
+            in the env vars or .env file.
+        base_url: The optional deployment base_url. If provided will override the value
+            in the env vars or .env file.
+        api_version: The optional deployment api version. If provided will override the value
+            in the env vars or .env file.
+        ad_token: The Azure AD token for authentication. (Optional)
+        ad_token_provider: Whether to use Azure Active Directory authentication.
+            (Optional) The default value is False.
+        token_endpoint: The Azure AD token endpoint. (Optional)
+        default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+        async_client (Optional[AsyncAzureOpenAI]): An existing client to use. (Optional)
+        env_file_path (str | None): Use the environment settings file as a fallback to
+            environment variables. (Optional)
+        credential (TokenCredential): The credential to use for authentication.
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                env_file_path=env_file_path,
+                api_key=api_key,
+                embedding_deployment_name=deployment_name,
+                endpoint=endpoint,
+                base_url=base_url,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as exc:
+            raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
+        if not azure_openai_settings.embedding_deployment_name:
+            raise ServiceInitializationError("The Azure OpenAI embedding deployment name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.embedding_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.EMBEDDING,
+            client=async_client,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls, settings: dict[str, Any]) -> "AzureTextEmbedding":
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: deployment_name, endpoint, api_key
+                and optionally: api_version, ad_auth
+        """
+        return AzureTextEmbedding(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/azure_text_to_audio.py

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio_base import OpenAITextToAudioBase
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="AzureTextToAudio")
+
+
+class AzureTextToAudio(AzureOpenAIConfigBase, OpenAITextToAudioBase):
+    """Azure text to audio service."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = "2024-10-01-preview",
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureTextToAudio service.
+
+        Args:
+            service_id: The service ID. (Optional)
+            api_key: The optional api key. If provided, will override the value in the
+                    env vars or .env file.
+            deployment_name: The optional deployment. If provided, will override the value
+                (text_to_audio_deployment_name) in the env vars or .env file.
+            endpoint: The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url: The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version: The optional deployment api version. If provided will override the value
+                in the env vars or .env file. Default is "2024-10-01-preview".
+            ad_token: The Azure AD token for authentication. (Optional)
+            ad_token_provider: Azure AD Token provider. (Optional)
+            token_endpoint: The Azure AD token endpoint. (Optional)
+            default_headers: The default headers mapping of string keys to
+                    string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+            credential: The credential to use for authentication. (Optional)
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+                api_key=api_key,
+                text_to_audio_deployment_name=deployment_name,
+                endpoint=endpoint,
+                base_url=base_url,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as exc:
+            raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
+        if not azure_openai_settings.text_to_audio_deployment_name:
+            raise ServiceInitializationError("The Azure OpenAI text to audio deployment name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.text_to_audio_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.TEXT_TO_AUDIO,
+            client=async_client,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: deployment_name, endpoint, api_key
+                and optionally: api_version, ad_auth
+        """
+        return cls(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/azure_text_to_image.py

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from azure.core.credentials import TokenCredential
+from openai import AsyncAzureOpenAI
+from openai.lib.azure import AsyncAzureADTokenProvider
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.azure_config_base import AzureOpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image_base import OpenAITextToImageBase
+from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="AzureTextToImage")
+
+
+class AzureTextToImage(AzureOpenAIConfigBase, OpenAITextToImageBase):
+    """Azure Text to Image service."""
+
+    def __init__(
+        self,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        deployment_name: str | None = None,
+        endpoint: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        ad_token: str | None = None,
+        ad_token_provider: AsyncAzureADTokenProvider | None = None,
+        token_endpoint: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncAzureOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        credential: TokenCredential | None = None,
+    ) -> None:
+        """Initialize an AzureTextToImage service.
+
+        Args:
+            service_id: The service ID. (Optional)
+            api_key: The optional api key. If provided, will override the value in the
+                    env vars or .env file.
+            deployment_name: The optional deployment. If provided, will override the value
+                (text_to_image_deployment_name) in the env vars or .env file.
+            endpoint: The optional deployment endpoint. If provided will override the value
+                in the env vars or .env file.
+            base_url: The optional deployment base_url. If provided will override the value
+                in the env vars or .env file.
+            api_version: The optional deployment api version. If provided will override the value
+                in the env vars or .env file.
+            ad_token: The Azure AD token for authentication. (Optional)
+            ad_token_provider: Azure AD Token provider. (Optional)
+            token_endpoint: The Azure AD token endpoint. (Optional)
+            default_headers: The default headers mapping of string keys to
+                    string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+            credential: The credential to use for authentication. (Optional)
+        """
+        try:
+            azure_openai_settings = AzureOpenAISettings(
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+                api_key=api_key,
+                text_to_image_deployment_name=deployment_name,
+                endpoint=endpoint,
+                base_url=base_url,
+                api_version=api_version,
+                token_endpoint=token_endpoint,
+            )
+        except ValidationError as exc:
+            raise ServiceInitializationError(f"Invalid settings: {exc}") from exc
+        if not azure_openai_settings.text_to_image_deployment_name:
+            raise ServiceInitializationError("The Azure OpenAI text to image deployment name is required.")
+
+        super().__init__(
+            deployment_name=azure_openai_settings.text_to_image_deployment_name,
+            endpoint=azure_openai_settings.endpoint,
+            base_url=azure_openai_settings.base_url,
+            api_version=azure_openai_settings.api_version,
+            service_id=service_id,
+            api_key=azure_openai_settings.api_key.get_secret_value() if azure_openai_settings.api_key else None,
+            ad_token=ad_token,
+            ad_token_provider=ad_token_provider,
+            token_endpoint=azure_openai_settings.token_endpoint,
+            default_headers=default_headers,
+            ai_model_type=OpenAIModelTypes.TEXT_TO_IMAGE,
+            client=async_client,
+            credential=credential,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Azure OpenAI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+                should contain keys: deployment_name, endpoint, api_key
+                and optionally: api_version, ad_auth
+        """
+        return cls(
+            service_id=settings.get("service_id"),
+            api_key=settings.get("api_key"),
+            deployment_name=settings.get("deployment_name"),
+            endpoint=settings.get("endpoint"),
+            base_url=settings.get("base_url"),
+            api_version=settings.get("api_version"),
+            ad_token=settings.get("ad_token"),
+            ad_token_provider=settings.get("ad_token_provider"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_audio_to_text.py

@@ -0,0 +1,85 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text_base import OpenAIAudioToTextBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="OpenAIAudioToText")
+
+
+class OpenAIAudioToText(OpenAIConfigBase, OpenAIAudioToTextBase):
+    """OpenAI Text to Image service."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """Initializes a new instance of the OpenAIAudioToText class.
+
+        Args:
+            ai_model_id: OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id: Service ID tied to the execution settings.
+            api_key: The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id: The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as
+                a fallback to environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                audio_to_text_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.audio_to_text_model_id:
+            raise ServiceInitializationError("The OpenAI audio to text model ID is required.")
+        super().__init__(
+            ai_model_id=openai_settings.audio_to_text_model_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            ai_model_type=OpenAIModelTypes.AUDIO_TO_TEXT,
+            org_id=openai_settings.org_id,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        return cls(
+            ai_model_id=settings.get("ai_model_id"),
+            api_key=settings.get("api_key"),
+            org_id=settings.get("org_id"),
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers", {}),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_audio_to_text_base.py

@@ -0,0 +1,63 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import sys
+from typing import Any
+
+from openai.types.audio import Transcription
+
+from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from semantic_kernel.connectors.ai.audio_to_text_client_base import AudioToTextClientBase
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_audio_to_text_execution_settings import (
+    OpenAIAudioToTextExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.contents import AudioContent, TextContent
+
+
+class OpenAIAudioToTextBase(OpenAIHandler, AudioToTextClientBase):
+    """OpenAI audio to text client."""
+
+    @override
+    async def get_text_contents(
+        self,
+        audio_content: AudioContent,
+        settings: PromptExecutionSettings | None = None,
+        **kwargs: Any,
+    ) -> list[TextContent]:
+        if not settings:
+            settings = OpenAIAudioToTextExecutionSettings(ai_model_id=self.ai_model_id)
+        else:
+            if not isinstance(settings, OpenAIAudioToTextExecutionSettings):
+                settings = self.get_prompt_execution_settings_from_settings(settings)
+
+        assert isinstance(settings, OpenAIAudioToTextExecutionSettings)  # nosec
+
+        if settings.ai_model_id is None:
+            settings.ai_model_id = self.ai_model_id
+
+        if not isinstance(audio_content.uri, str):
+            raise ServiceInvalidRequestError("Audio content uri must be a string to a local file.")
+
+        settings.filename = audio_content.uri
+
+        response = await self._send_request(settings)
+        assert isinstance(response, Transcription)  # nosec
+
+        return [
+            TextContent(
+                ai_model_id=settings.ai_model_id,
+                text=response.text,
+                inner_content=response,
+            )
+        ]
+
+    def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
+        """Get the request settings class."""
+        return OpenAIAudioToTextExecutionSettings

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion.py

@@ -0,0 +1,91 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion_base import OpenAIChatCompletionBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class OpenAIChatCompletion(OpenAIConfigBase, OpenAIChatCompletionBase, OpenAITextCompletionBase):
+    """OpenAI Chat completion class."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        service_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        instruction_role: str | None = None,
+    ) -> None:
+        """Initialize an OpenAIChatCompletion service.
+
+        Args:
+            ai_model_id (str): OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id (str | None): Service ID tied to the execution settings.
+            api_key (str | None): The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id (str | None): The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback
+                to environment variables. (Optional)
+            env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
+            instruction_role (str | None): The role to use for 'instruction' messages, for example,
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                chat_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+
+        if not async_client and not openai_settings.api_key:
+            raise ServiceInitializationError("The OpenAI API key is required.")
+        if not openai_settings.chat_model_id:
+            raise ServiceInitializationError("The OpenAI model ID is required.")
+
+        super().__init__(
+            ai_model_id=openai_settings.chat_model_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            org_id=openai_settings.org_id,
+            service_id=service_id,
+            ai_model_type=OpenAIModelTypes.CHAT,
+            default_headers=default_headers,
+            client=async_client,
+            instruction_role=instruction_role,
+        )
+
+    @classmethod
+    def from_dict(cls, settings: dict[str, Any]) -> "OpenAIChatCompletion":
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        return OpenAIChatCompletion(
+            ai_model_id=settings["ai_model_id"],
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_chat_completion_base.py

@@ -0,0 +1,329 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import sys
+from collections.abc import AsyncGenerator, Callable
+from typing import TYPE_CHECKING, Any, ClassVar, cast
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from openai import AsyncStream
+from openai.types.chat.chat_completion import ChatCompletion, Choice
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk, ChoiceDeltaFunctionCall, ChoiceDeltaToolCall
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from openai.types.chat.chat_completion_message import FunctionCall
+from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall
+from typing_extensions import deprecated
+
+from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
+from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
+from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration
+from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior, FunctionChoiceType
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
+    OpenAIChatPromptExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.contents.annotation_content import AnnotationContent
+from semantic_kernel.contents.chat_history import ChatHistory
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.file_reference_content import FileReferenceContent
+from semantic_kernel.contents.function_call_content import FunctionCallContent
+from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
+from semantic_kernel.contents.streaming_text_content import StreamingTextContent
+from semantic_kernel.contents.text_content import TextContent
+from semantic_kernel.contents.utils.author_role import AuthorRole
+from semantic_kernel.contents.utils.finish_reason import FinishReason
+from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError, ServiceInvalidResponseError
+from semantic_kernel.filters.auto_function_invocation.auto_function_invocation_context import (
+    AutoFunctionInvocationContext,
+)
+from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
+    trace_chat_completion,
+    trace_streaming_chat_completion,
+)
+
+if TYPE_CHECKING:
+    from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
+    from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+
+class OpenAIChatCompletionBase(OpenAIHandler, ChatCompletionClientBase):
+    """OpenAI Chat completion class."""
+
+    MODEL_PROVIDER_NAME: ClassVar[str] = "openai"
+    SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True
+
+    # region Overriding base class methods
+    # most of the methods are overridden from the ChatCompletionClientBase class, otherwise it is mentioned
+
+    # Override from AIServiceClientBase
+    @override
+    def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
+        return OpenAIChatPromptExecutionSettings
+
+    # Override from AIServiceClientBase
+    @override
+    def service_url(self) -> str | None:
+        return str(self.client.base_url)
+
+    @override
+    @trace_chat_completion(MODEL_PROVIDER_NAME)
+    async def _inner_get_chat_message_contents(
+        self,
+        chat_history: "ChatHistory",
+        settings: "PromptExecutionSettings",
+    ) -> list["ChatMessageContent"]:
+        if not isinstance(settings, OpenAIChatPromptExecutionSettings):
+            settings = self.get_prompt_execution_settings_from_settings(settings)
+        assert isinstance(settings, OpenAIChatPromptExecutionSettings)  # nosec
+
+        settings.stream = False
+        settings.messages = self._prepare_chat_history_for_request(chat_history)
+        settings.ai_model_id = settings.ai_model_id or self.ai_model_id
+
+        response = await self._send_request(settings)
+        assert isinstance(response, ChatCompletion)  # nosec
+        response_metadata = self._get_metadata_from_chat_response(response)
+        return [self._create_chat_message_content(response, choice, response_metadata) for choice in response.choices]
+
+    @override
+    @trace_streaming_chat_completion(MODEL_PROVIDER_NAME)
+    async def _inner_get_streaming_chat_message_contents(
+        self,
+        chat_history: "ChatHistory",
+        settings: "PromptExecutionSettings",
+        function_invoke_attempt: int = 0,
+    ) -> AsyncGenerator[list["StreamingChatMessageContent"], Any]:
+        if not isinstance(settings, OpenAIChatPromptExecutionSettings):
+            settings = self.get_prompt_execution_settings_from_settings(settings)
+        assert isinstance(settings, OpenAIChatPromptExecutionSettings)  # nosec
+
+        settings.stream = True
+        settings.stream_options = {"include_usage": True}
+        settings.messages = self._prepare_chat_history_for_request(chat_history)
+        settings.ai_model_id = settings.ai_model_id or self.ai_model_id
+
+        response = await self._send_request(settings)
+        if not isinstance(response, AsyncStream):
+            raise ServiceInvalidResponseError("Expected an AsyncStream[ChatCompletionChunk] response.")
+        async for chunk in response:
+            if len(chunk.choices) == 0 and chunk.usage is None:
+                continue
+
+            assert isinstance(chunk, ChatCompletionChunk)  # nosec
+            chunk_metadata = self._get_metadata_from_streaming_chat_response(chunk)
+            if (not chunk.choices or len(chunk.choices) == 0) and chunk.usage is not None:
+                # Usage is contained in the last chunk where the choices are empty
+                # We are duplicating the usage metadata to all the choices in the response
+                yield [
+                    StreamingChatMessageContent(
+                        role=AuthorRole.ASSISTANT,
+                        content="",
+                        choice_index=i,
+                        inner_content=chunk,
+                        ai_model_id=settings.ai_model_id,
+                        metadata=chunk_metadata,
+                        function_invoke_attempt=function_invoke_attempt,
+                    )
+                    for i in range(settings.number_of_responses or 1)
+                ]
+            else:
+                yield [
+                    self._create_streaming_chat_message_content(chunk, choice, chunk_metadata, function_invoke_attempt)
+                    for choice in chunk.choices
+                ]
+
+    @override
+    def _verify_function_choice_settings(self, settings: "PromptExecutionSettings") -> None:
+        if not isinstance(settings, OpenAIChatPromptExecutionSettings):
+            raise ServiceInvalidExecutionSettingsError("The settings must be an OpenAIChatPromptExecutionSettings.")
+        if settings.number_of_responses is not None and settings.number_of_responses > 1:
+            raise ServiceInvalidExecutionSettingsError(
+                "Auto-invocation of tool calls may only be used with a "
+                "OpenAIChatPromptExecutions.number_of_responses of 1."
+            )
+
+    @override
+    def _update_function_choice_settings_callback(
+        self,
+    ) -> Callable[["FunctionCallChoiceConfiguration", "PromptExecutionSettings", FunctionChoiceType], None]:
+        return update_settings_from_function_call_configuration
+
+    @override
+    def _reset_function_choice_settings(self, settings: "PromptExecutionSettings") -> None:
+        if hasattr(settings, "tool_choice"):
+            settings.tool_choice = None
+        if hasattr(settings, "tools"):
+            settings.tools = None
+
+    # endregion
+
+    # region content creation
+
+    def _create_chat_message_content(
+        self, response: ChatCompletion, choice: Choice, response_metadata: dict[str, Any]
+    ) -> "ChatMessageContent":
+        """Create a chat message content object from a choice."""
+        metadata = self._get_metadata_from_chat_choice(choice)
+        metadata.update(response_metadata)
+
+        items: list[Any] = self._get_tool_calls_from_chat_choice(choice)
+        items.extend(self._get_function_call_from_chat_choice(choice))
+        if choice.message.content:
+            items.append(TextContent(text=choice.message.content))
+        elif hasattr(choice.message, "refusal") and choice.message.refusal:
+            items.append(TextContent(text=choice.message.refusal))
+
+        return ChatMessageContent(
+            inner_content=response,
+            ai_model_id=self.ai_model_id,
+            metadata=metadata,
+            role=AuthorRole(choice.message.role),
+            items=items,
+            finish_reason=(FinishReason(choice.finish_reason) if choice.finish_reason else None),
+        )
+
+    def _create_streaming_chat_message_content(
+        self,
+        chunk: ChatCompletionChunk,
+        choice: ChunkChoice,
+        chunk_metadata: dict[str, Any],
+        function_invoke_attempt: int,
+    ) -> StreamingChatMessageContent:
+        """Create a streaming chat message content object from a choice."""
+        metadata = self._get_metadata_from_chat_choice(choice)
+        metadata.update(chunk_metadata)
+
+        items: list[Any] = self._get_tool_calls_from_chat_choice(choice)
+        items.extend(self._get_function_call_from_chat_choice(choice))
+        if choice.delta and choice.delta.content is not None:
+            items.append(StreamingTextContent(choice_index=choice.index, text=choice.delta.content))
+        return StreamingChatMessageContent(
+            choice_index=choice.index,
+            inner_content=chunk,
+            ai_model_id=self.ai_model_id,
+            metadata=metadata,
+            role=(AuthorRole(choice.delta.role) if choice.delta and choice.delta.role else AuthorRole.ASSISTANT),
+            finish_reason=(FinishReason(choice.finish_reason) if choice.finish_reason else None),
+            items=items,
+            function_invoke_attempt=function_invoke_attempt,
+        )
+
+    def _get_metadata_from_chat_response(self, response: ChatCompletion) -> dict[str, Any]:
+        """Get metadata from a chat response."""
+        return {
+            "id": response.id,
+            "created": response.created,
+            "system_fingerprint": response.system_fingerprint,
+            "usage": CompletionUsage.from_openai(response.usage) if response.usage is not None else None,
+        }
+
+    def _get_metadata_from_streaming_chat_response(self, response: ChatCompletionChunk) -> dict[str, Any]:
+        """Get metadata from a streaming chat response."""
+        return {
+            "id": response.id,
+            "created": response.created,
+            "system_fingerprint": response.system_fingerprint,
+            "usage": CompletionUsage.from_openai(response.usage) if response.usage is not None else None,
+        }
+
+    def _get_metadata_from_chat_choice(self, choice: Choice | ChunkChoice) -> dict[str, Any]:
+        """Get metadata from a chat choice."""
+        return {
+            "logprobs": getattr(choice, "logprobs", None),
+        }
+
+    def _get_tool_calls_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]:
+        """Get tool calls from a chat choice."""
+        content = choice.message if isinstance(choice, Choice) else choice.delta
+        if content and (tool_calls := getattr(content, "tool_calls", None)) is not None:
+            return [
+                FunctionCallContent(
+                    id=tool.id,
+                    index=getattr(tool, "index", None),
+                    name=tool.function.name,
+                    arguments=tool.function.arguments,
+                )
+                for tool in cast(list[ChatCompletionMessageToolCall] | list[ChoiceDeltaToolCall], tool_calls)
+                if tool.function is not None
+            ]
+        # When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
+        return []
+
+    def _get_function_call_from_chat_choice(self, choice: Choice | ChunkChoice) -> list[FunctionCallContent]:
+        """Get a function call from a chat choice."""
+        content = choice.message if isinstance(choice, Choice) else choice.delta
+        if content and (function_call := getattr(content, "function_call", None)) is not None:
+            function_call = cast(FunctionCall | ChoiceDeltaFunctionCall, function_call)
+            return [
+                FunctionCallContent(
+                    id="legacy_function_call", name=function_call.name, arguments=function_call.arguments
+                )
+            ]
+        # When you enable asynchronous content filtering in Azure OpenAI, you may receive empty deltas
+        return []
+
+    def _prepare_chat_history_for_request(
+        self,
+        chat_history: "ChatHistory",
+        role_key: str = "role",
+        content_key: str = "content",
+    ) -> Any:
+        """Prepare the chat history for a request.
+
+        Allowing customization of the key names for role/author, and optionally overriding the role.
+
+        ChatRole.TOOL messages need to be formatted different than system/user/assistant messages:
+            They require a "tool_call_id" and (function) "name" key, and the "metadata" key should
+            be removed. The "encoding" key should also be removed.
+
+        Override this method to customize the formatting of the chat history for a request.
+
+        Args:
+            chat_history (ChatHistory): The chat history to prepare.
+            role_key (str): The key name for the role/author.
+            content_key (str): The key name for the content/message.
+
+        Returns:
+            prepared_chat_history (Any): The prepared chat history for a request.
+        """
+        return [
+            {
+                **message.to_dict(role_key=role_key, content_key=content_key),
+                role_key: "developer"
+                if self.instruction_role == "developer" and message.to_dict(role_key=role_key)[role_key] == "system"
+                else message.to_dict(role_key=role_key)[role_key],
+            }
+            for message in chat_history.messages
+            if not isinstance(message, (AnnotationContent, FileReferenceContent))
+        ]
+
+    # endregion
+
+    # region function calling
+    @deprecated("Use `invoke_function_call` from the kernel instead with `FunctionChoiceBehavior`.")
+    async def _process_function_call(
+        self,
+        function_call: FunctionCallContent,
+        chat_history: ChatHistory,
+        kernel: "Kernel",
+        arguments: "KernelArguments | None",
+        function_call_count: int,
+        request_index: int,
+        function_call_behavior: FunctionChoiceBehavior,
+    ) -> "AutoFunctionInvocationContext | None":
+        """Processes the tool calls in the result and update the chat history."""
+        return await kernel.invoke_function_call(
+            function_call=function_call,
+            chat_history=chat_history,
+            arguments=arguments,
+            function_call_count=function_call_count,
+            request_index=request_index,
+            function_behavior=function_call_behavior,
+        )
+
+    # endregion

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py

@@ -0,0 +1,106 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Mapping
+from copy import copy
+from typing import Any
+
+from openai import AsyncOpenAI
+from pydantic import ConfigDict, Field, validate_call
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.const import USER_AGENT
+from semantic_kernel.exceptions import ServiceInitializationError
+from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class OpenAIConfigBase(OpenAIHandler):
+    """Internal class for configuring a connection to an OpenAI service."""
+
+    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
+    def __init__(
+        self,
+        ai_model_id: str = Field(min_length=1),
+        api_key: str | None = Field(min_length=1),
+        ai_model_type: OpenAIModelTypes | None = OpenAIModelTypes.CHAT,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        client: AsyncOpenAI | None = None,
+        instruction_role: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize a client for OpenAI services.
+
+        This constructor sets up a client to interact with OpenAI's API, allowing for
+        different types of AI model interactions, like chat or text completion.
+
+        Args:
+            ai_model_id (str): OpenAI model identifier. Must be non-empty.
+                Default to a preset value.
+            api_key (str): OpenAI API key for authentication.
+                Must be non-empty. (Optional)
+            ai_model_type (OpenAIModelTypes): The type of OpenAI
+                model to interact with. Defaults to CHAT.
+            org_id (str): OpenAI organization ID. This is optional
+                unless the account belongs to multiple organizations.
+            service_id (str): OpenAI service ID. This is optional.
+            default_headers (Mapping[str, str]): Default headers
+                for HTTP requests. (Optional)
+            client (AsyncOpenAI): An existing OpenAI client, optional.
+            instruction_role (str): The role to use for 'instruction'
+                messages, for example, summarization prompts could use `developer` or `system`. (Optional)
+            kwargs: Additional keyword arguments.
+
+        """
+        # Merge APP_INFO into the headers if it exists
+        merged_headers = dict(copy(default_headers)) if default_headers else {}
+        if APP_INFO:
+            merged_headers.update(APP_INFO)
+            merged_headers = prepend_semantic_kernel_to_user_agent(merged_headers)
+
+        if not client:
+            if not api_key:
+                raise ServiceInitializationError("Please provide an api_key")
+            client = AsyncOpenAI(
+                api_key=api_key,
+                organization=org_id,
+                default_headers=merged_headers,
+            )
+        args = {
+            "ai_model_id": ai_model_id,
+            "client": client,
+            "ai_model_type": ai_model_type,
+        }
+        if service_id:
+            args["service_id"] = service_id
+        if instruction_role:
+            args["instruction_role"] = instruction_role
+        super().__init__(**args, **kwargs)
+
+    def to_dict(self) -> dict[str, str]:
+        """Create a dict of the service settings."""
+        client_settings = {
+            "api_key": self.client.api_key,
+            "default_headers": {k: v for k, v in self.client.default_headers.items() if k != USER_AGENT},
+        }
+        if self.client.organization:
+            client_settings["org_id"] = self.client.organization
+        base = self.model_dump(
+            exclude={
+                "prompt_tokens",
+                "completion_tokens",
+                "total_tokens",
+                "api_type",
+                "ai_model_type",
+                "service_id",
+                "client",
+            },
+            by_alias=True,
+            exclude_none=True,
+        )
+        base.update(client_settings)
+        return base

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_handler.py

@@ -0,0 +1,235 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from abc import ABC
+from typing import Any, Union
+
+from openai import AsyncOpenAI, AsyncStream, BadRequestError, _legacy_response
+from openai._types import FileTypes, Omit, omit
+from openai.lib._parsing._completions import type_to_response_format_param
+from openai.types import Completion, CreateEmbeddingResponse
+from openai.types.audio import Transcription
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
+from openai.types.images_response import ImagesResponse
+from pydantic import BaseModel
+
+from semantic_kernel.connectors.ai.open_ai import (
+    OpenAIAudioToTextExecutionSettings,
+    OpenAIChatPromptExecutionSettings,
+    OpenAIEmbeddingPromptExecutionSettings,
+    OpenAIPromptExecutionSettings,
+    OpenAITextToAudioExecutionSettings,
+    OpenAITextToImageExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import ContentFilterAIException
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.connectors.utils.structured_output_schema import generate_structured_output_response_format_schema
+from semantic_kernel.exceptions import ServiceResponseException
+from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+from semantic_kernel.schema.kernel_json_schema_builder import KernelJsonSchemaBuilder
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+RESPONSE_TYPE = Union[
+    ChatCompletion,
+    Completion,
+    AsyncStream[ChatCompletionChunk],
+    AsyncStream[Completion],
+    list[Any],
+    ImagesResponse,
+    Transcription,
+    _legacy_response.HttpxBinaryResponseContent,
+]
+
+
+class OpenAIHandler(KernelBaseModel, ABC):
+    """Internal class for calls to OpenAI API's."""
+
+    client: AsyncOpenAI
+    ai_model_type: OpenAIModelTypes = OpenAIModelTypes.CHAT
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+
+    async def _send_request(self, settings: PromptExecutionSettings) -> RESPONSE_TYPE:
+        """Send a request to the OpenAI API."""
+        if self.ai_model_type == OpenAIModelTypes.TEXT or self.ai_model_type == OpenAIModelTypes.CHAT:
+            assert isinstance(settings, OpenAIPromptExecutionSettings)  # nosec
+            return await self._send_completion_request(settings)
+        if self.ai_model_type == OpenAIModelTypes.EMBEDDING:
+            assert isinstance(settings, OpenAIEmbeddingPromptExecutionSettings)  # nosec
+            return await self._send_embedding_request(settings)
+        if self.ai_model_type == OpenAIModelTypes.TEXT_TO_IMAGE:
+            assert isinstance(settings, OpenAITextToImageExecutionSettings)  # nosec
+            return await self._send_text_to_image_request(settings)
+        if self.ai_model_type == OpenAIModelTypes.AUDIO_TO_TEXT:
+            assert isinstance(settings, OpenAIAudioToTextExecutionSettings)  # nosec
+            return await self._send_audio_to_text_request(settings)
+        if self.ai_model_type == OpenAIModelTypes.TEXT_TO_AUDIO:
+            assert isinstance(settings, OpenAITextToAudioExecutionSettings)  # nosec
+            return await self._send_text_to_audio_request(settings)
+
+        raise NotImplementedError(f"Model type {self.ai_model_type} is not supported")
+
+    async def _send_completion_request(
+        self,
+        settings: OpenAIPromptExecutionSettings,
+    ) -> ChatCompletion | Completion | AsyncStream[ChatCompletionChunk] | AsyncStream[Completion]:
+        """Execute the appropriate call to OpenAI models."""
+        try:
+            settings_dict = settings.prepare_settings_dict()
+            if self.ai_model_type == OpenAIModelTypes.CHAT:
+                assert isinstance(settings, OpenAIChatPromptExecutionSettings)  # nosec
+                self._handle_structured_output(settings, settings_dict)
+                if settings.tools is None:
+                    settings_dict.pop("parallel_tool_calls", None)
+                response = await self.client.chat.completions.create(**settings_dict)
+            else:
+                response = await self.client.completions.create(**settings_dict)
+
+            self.store_usage(response)
+            return response
+        except BadRequestError as ex:
+            if ex.code == "content_filter":
+                raise ContentFilterAIException(
+                    f"{type(self)} service encountered a content error",
+                    ex,
+                ) from ex
+            raise ServiceResponseException(
+                f"{type(self)} service failed to complete the prompt",
+                ex,
+            ) from ex
+        except Exception as ex:
+            raise ServiceResponseException(
+                f"{type(self)} service failed to complete the prompt",
+                ex,
+            ) from ex
+
+    async def _send_embedding_request(self, settings: OpenAIEmbeddingPromptExecutionSettings) -> list[Any]:
+        """Send a request to the OpenAI embeddings endpoint."""
+        try:
+            response = await self.client.embeddings.create(**settings.prepare_settings_dict())
+
+            self.store_usage(response)
+            return [x.embedding for x in response.data]
+        except Exception as ex:
+            raise ServiceResponseException(
+                f"{type(self)} service failed to generate embeddings",
+                ex,
+            ) from ex
+
+    async def _send_text_to_image_request(self, settings: OpenAITextToImageExecutionSettings) -> ImagesResponse:
+        """Send a request to the OpenAI text to image endpoint."""
+        try:
+            response: ImagesResponse = await self.client.images.generate(
+                **settings.prepare_settings_dict(),
+            )
+            self.store_usage(response)
+            return response
+        except Exception as ex:
+            raise ServiceResponseException(f"Failed to generate image: {ex}") from ex
+
+    async def _send_image_edit_request(
+        self,
+        image: list[FileTypes],
+        settings: OpenAITextToImageExecutionSettings,
+        mask: FileTypes | Omit = omit,
+    ) -> ImagesResponse:
+        """Send a request to the OpenAI image edit endpoint.
+
+        Args:
+            image: List of image files to edit. Accepts file paths or bytes.
+            settings: Image edit execution settings.
+            mask: Optional mask image. Accepts file path or bytes.
+
+        Returns:
+            ImagesResponse: The response from the image edit API.
+        """
+        try:
+            response: ImagesResponse = await self.client.images.edit(
+                image=image,
+                mask=mask,  # type: ignore
+                **settings.prepare_settings_dict(),
+            )
+            self.store_usage(response)
+            return response
+        except Exception as ex:
+            raise ServiceResponseException(f"Failed to edit image: {ex}") from ex
+
+    async def _send_audio_to_text_request(self, settings: OpenAIAudioToTextExecutionSettings) -> Transcription:
+        """Send a request to the OpenAI audio to text endpoint."""
+        if not settings.filename:
+            raise ServiceInvalidRequestError("Audio file is required for audio to text service")
+
+        try:
+            with open(settings.filename, "rb") as audio_file:
+                return await self.client.audio.transcriptions.create(
+                    file=audio_file,
+                    **settings.prepare_settings_dict(),
+                )
+        except Exception as ex:
+            raise ServiceResponseException(
+                f"{type(self)} service failed to transcribe audio",
+                ex,
+            ) from ex
+
+    async def _send_text_to_audio_request(
+        self, settings: OpenAITextToAudioExecutionSettings
+    ) -> _legacy_response.HttpxBinaryResponseContent:
+        """Send a request to the OpenAI text to audio endpoint.
+
+        The OpenAI API returns the content of the generated audio file.
+        """
+        try:
+            return await self.client.audio.speech.create(
+                **settings.prepare_settings_dict(),
+            )
+        except Exception as ex:
+            raise ServiceResponseException(
+                f"{type(self)} service failed to generate audio",
+                ex,
+            ) from ex
+
+    def _handle_structured_output(
+        self, request_settings: OpenAIChatPromptExecutionSettings, settings: dict[str, Any]
+    ) -> None:
+        response_format = getattr(request_settings, "response_format", None)
+        if getattr(request_settings, "structured_json_response", False) and response_format:
+            # Case 1: response_format is a type and subclass of BaseModel
+            if isinstance(response_format, type) and issubclass(response_format, BaseModel):
+                settings["response_format"] = type_to_response_format_param(response_format)
+            # Case 2: response_format is a type but not a subclass of BaseModel
+            elif isinstance(response_format, type):
+                generated_schema = KernelJsonSchemaBuilder.build(parameter_type=response_format, structured_output=True)
+                assert generated_schema is not None  # nosec
+                settings["response_format"] = generate_structured_output_response_format_schema(
+                    name=response_format.__name__, schema=generated_schema
+                )
+            # Case 3: response_format is a dictionary, pass it without modification
+            elif isinstance(response_format, dict):
+                settings["response_format"] = response_format
+
+    def store_usage(
+        self,
+        response: ChatCompletion
+        | Completion
+        | AsyncStream[ChatCompletionChunk]
+        | AsyncStream[Completion]
+        | CreateEmbeddingResponse
+        | ImagesResponse,
+    ):
+        """Store the usage information from the response."""
+        if isinstance(response, ImagesResponse) and hasattr(response, "usage") and response.usage:
+            logger.info(f"OpenAI image usage: {response.usage}")
+            self.prompt_tokens += response.usage.input_tokens
+            self.total_tokens += response.usage.total_tokens
+            self.completion_tokens += response.usage.output_tokens
+            return
+        if not isinstance(response, AsyncStream) and not isinstance(response, ImagesResponse) and response.usage:
+            logger.info(f"OpenAI usage: {response.usage}")
+            self.prompt_tokens += response.usage.prompt_tokens
+            self.total_tokens += response.usage.total_tokens
+            if hasattr(response.usage, "completion_tokens"):
+                self.completion_tokens += response.usage.completion_tokens  # type: ignore

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_model_types.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from enum import Enum
+
+
+class OpenAIModelTypes(Enum):
+    """OpenAI model types, can be text, chat or embedding."""
+
+    TEXT = "text"
+    CHAT = "chat"
+    EMBEDDING = "embedding"
+    TEXT_TO_IMAGE = "text-to-image"
+    AUDIO_TO_TEXT = "audio-to-text"
+    TEXT_TO_AUDIO = "text-to-audio"
+    REALTIME = "realtime"
+    RESPONSE = "response"

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py

@@ -0,0 +1,190 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Callable, Coroutine, Mapping
+from typing import TYPE_CHECKING, Any
+
+from numpy import ndarray
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import (
+    ListenEvents,
+    OpenAIRealtimeWebRTCBase,
+    OpenAIRealtimeWebsocketBase,
+    SendEvents,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.utils.feature_stage_decorator import experimental
+
+if TYPE_CHECKING:
+    from aiortc.mediastreams import MediaStreamTrack
+    from numpy import ndarray
+
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+__all__ = [
+    "ListenEvents",
+    "OpenAIRealtimeWebRTC",
+    "OpenAIRealtimeWebsocket",
+    "SendEvents",
+]
+
+
+@experimental
+class OpenAIRealtimeWebRTC(OpenAIRealtimeWebRTCBase, OpenAIConfigBase):
+    """OpenAI Realtime service using WebRTC protocol."""
+
+    def __init__(
+        self,
+        audio_track: "MediaStreamTrack",
+        audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize an OpenAIRealtime service.
+
+        Args:
+            audio_track: The audio track to use for the service, only used by WebRTC.
+                It can be any class that implements the AudioStreamTrack interface.
+            audio_output_callback: The audio output callback, optional.
+                This should be a coroutine, that takes a ndarray with audio as input.
+                The goal of this function is to allow you to play the audio with the
+                least amount of latency possible, because it is called first before further processing.
+                It can also be set in the `receive` method.
+                Even when passed, the audio content will still be
+                added to the receiving queue.
+            ai_model_id (str | None): OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id (str | None): Service ID tied to the execution settings.
+            api_key (str | None): The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id (str | None): The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
+            kwargs: Additional arguments.
+                This can include:
+                kernel (Kernel): the kernel to use for function calls
+                plugins (list[object] or dict[str, object]): the plugins to use for function calls
+                settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
+                chat_history (ChatHistory): the chat history to use for the session
+                Otherwise they can also be passed to the context manager.
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                realtime_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.realtime_model_id:
+            raise ServiceInitializationError("The OpenAI realtime model ID is required.")
+        if audio_track:
+            kwargs["audio_track"] = audio_track
+        super().__init__(
+            audio_output_callback=audio_output_callback,
+            ai_model_id=openai_settings.realtime_model_id,
+            service_id=service_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            org_id=openai_settings.org_id,
+            ai_model_type=OpenAIModelTypes.REALTIME,
+            default_headers=default_headers,
+            client=client,
+            **kwargs,
+        )
+
+
+# region Websocket
+
+
+@experimental
+class OpenAIRealtimeWebsocket(OpenAIRealtimeWebsocketBase, OpenAIConfigBase):
+    """OpenAI Realtime service using WebSocket protocol."""
+
+    def __init__(
+        self,
+        audio_output_callback: Callable[["ndarray"], Coroutine[Any, Any, None]] | None = None,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initialize an OpenAIRealtime service.
+
+        Args:
+            audio_output_callback: The audio output callback, optional.
+                This should be a coroutine, that takes a ndarray with audio as input.
+                The goal of this function is to allow you to play the audio with the
+                least amount of latency possible, because it is called first before further processing.
+                It can also be set in the `receive` method.
+                Even when passed, the audio content will still be
+                added to the receiving queue.
+            ai_model_id (str | None): OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id (str | None): Service ID tied to the execution settings.
+            api_key (str | None): The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id (str | None): The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
+            kwargs: Additional arguments.
+            kwargs: Additional arguments.
+                This can include:
+                kernel (Kernel): the kernel to use for function calls
+                plugins (list[object] or dict[str, object]): the plugins to use for function calls
+                settings (OpenAIRealtimeExecutionSettings): the settings to use for the session
+                chat_history (ChatHistory): the chat history to use for the session
+                Otherwise they can also be passed to the context manager.
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                realtime_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.realtime_model_id:
+            raise ServiceInitializationError("The OpenAI realtime model ID is required.")
+        super().__init__(
+            audio_output_callback=audio_output_callback,
+            ai_model_id=openai_settings.realtime_model_id,
+            service_id=service_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            org_id=openai_settings.org_id,
+            ai_model_type=OpenAIModelTypes.REALTIME,
+            default_headers=default_headers,
+            client=client,
+            **kwargs,
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion.py

@@ -0,0 +1,89 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import json
+import logging
+from collections.abc import Mapping
+from typing import Any
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion_base import OpenAITextCompletionBase
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class OpenAITextCompletion(OpenAITextCompletionBase, OpenAIConfigBase):
+    """OpenAI Text Completion class."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """Initialize an OpenAITextCompletion service.
+
+        Args:
+            ai_model_id (str | None): OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id (str | None): Service ID tied to the execution settings.
+            api_key (str | None): The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id (str | None): The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as a fallback to
+                environment variables. (Optional)
+            env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                text_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.text_model_id:
+            raise ServiceInitializationError("The OpenAI text model ID is required.")
+        super().__init__(
+            ai_model_id=openai_settings.text_model_id,
+            service_id=service_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            org_id=openai_settings.org_id,
+            ai_model_type=OpenAIModelTypes.TEXT,
+            default_headers=default_headers,
+            client=async_client,
+        )
+
+    @classmethod
+    def from_dict(cls, settings: dict[str, Any]) -> "OpenAITextCompletion":
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        if "default_headers" in settings and isinstance(settings["default_headers"], str):
+            settings["default_headers"] = json.loads(settings["default_headers"])
+        return OpenAITextCompletion(
+            ai_model_id=settings.get("ai_model_id"),
+            api_key=settings.get("api_key"),
+            org_id=settings.get("org_id"),
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers"),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_completion_base.py

@@ -0,0 +1,170 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+import sys
+from collections.abc import AsyncGenerator
+from typing import TYPE_CHECKING, Any, ClassVar
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from openai import AsyncStream
+from openai.types import Completion as TextCompletion
+from openai.types import CompletionChoice as TextCompletionChoice
+from openai.types.chat.chat_completion import ChatCompletion
+from openai.types.chat.chat_completion import Choice as ChatCompletionChoice
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
+from openai.types.chat.chat_completion_chunk import Choice as ChatCompletionChunkChoice
+
+from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
+    OpenAIChatPromptExecutionSettings,
+    OpenAITextPromptExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
+from semantic_kernel.contents.streaming_text_content import StreamingTextContent
+from semantic_kernel.contents.text_content import TextContent
+from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
+    trace_streaming_text_completion,
+    trace_text_completion,
+)
+
+if TYPE_CHECKING:
+    from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class OpenAITextCompletionBase(OpenAIHandler, TextCompletionClientBase):
+    """Base class for OpenAI text completion services."""
+
+    MODEL_PROVIDER_NAME: ClassVar[str] = "openai"
+
+    # region Overriding base class methods
+
+    # Override from AIServiceClientBase
+    @override
+    def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
+        return OpenAITextPromptExecutionSettings
+
+    # Override from AIServiceClientBase
+    @override
+    def service_url(self) -> str | None:
+        return str(self.client.base_url)
+
+    @override
+    @trace_text_completion(MODEL_PROVIDER_NAME)
+    async def _inner_get_text_contents(
+        self,
+        prompt: str,
+        settings: "PromptExecutionSettings",
+    ) -> list["TextContent"]:
+        if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)):
+            settings = self.get_prompt_execution_settings_from_settings(settings)
+        assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings))  # nosec
+
+        if isinstance(settings, OpenAITextPromptExecutionSettings):
+            settings.prompt = prompt
+        else:
+            settings.messages = [{"role": "user", "content": prompt}]
+
+        settings.ai_model_id = settings.ai_model_id or self.ai_model_id
+
+        response = await self._send_request(settings)
+        assert isinstance(response, (TextCompletion, ChatCompletion))  # nosec
+
+        metadata = self._get_metadata_from_text_response(response)
+        return [self._create_text_content(response, choice, metadata) for choice in response.choices]
+
+    @override
+    @trace_streaming_text_completion(MODEL_PROVIDER_NAME)
+    async def _inner_get_streaming_text_contents(
+        self,
+        prompt: str,
+        settings: "PromptExecutionSettings",
+    ) -> AsyncGenerator[list["StreamingTextContent"], Any]:
+        if not isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings)):
+            settings = self.get_prompt_execution_settings_from_settings(settings)
+        assert isinstance(settings, (OpenAITextPromptExecutionSettings, OpenAIChatPromptExecutionSettings))  # nosec
+
+        if isinstance(settings, OpenAITextPromptExecutionSettings):
+            settings.prompt = prompt
+        else:
+            if not settings.messages:
+                settings.messages = [{"role": "user", "content": prompt}]
+            else:
+                settings.messages.append({"role": "user", "content": prompt})
+
+        settings.ai_model_id = settings.ai_model_id or self.ai_model_id
+        settings.stream = True
+
+        response = await self._send_request(settings)
+        assert isinstance(response, AsyncStream)  # nosec
+
+        async for chunk in response:
+            if len(chunk.choices) == 0:
+                continue
+            assert isinstance(chunk, (TextCompletion, ChatCompletionChunk))  # nosec
+            chunk_metadata = self._get_metadata_from_text_response(chunk)
+            yield [self._create_streaming_text_content(chunk, choice, chunk_metadata) for choice in chunk.choices]
+
+    # endregion
+
+    def _create_text_content(
+        self,
+        response: TextCompletion | ChatCompletion,
+        choice: TextCompletionChoice | ChatCompletionChoice,
+        response_metadata: dict[str, Any],
+    ) -> "TextContent":
+        """Create a text content object from a choice."""
+        choice_metadata = self._get_metadata_from_text_choice(choice)
+        choice_metadata.update(response_metadata)
+        text = choice.text if isinstance(choice, TextCompletionChoice) else choice.message.content
+        return TextContent(
+            inner_content=response,
+            ai_model_id=self.ai_model_id,
+            text=text or "",
+            metadata=choice_metadata,
+        )
+
+    def _create_streaming_text_content(
+        self,
+        chunk: TextCompletion | ChatCompletionChunk,
+        choice: TextCompletionChoice | ChatCompletionChunkChoice,
+        response_metadata: dict[str, Any],
+    ) -> "StreamingTextContent":
+        """Create a streaming text content object from a choice."""
+        choice_metadata = self._get_metadata_from_text_choice(choice)
+        choice_metadata.update(response_metadata)
+        text = choice.text if isinstance(choice, TextCompletionChoice) else choice.delta.content
+        return StreamingTextContent(
+            choice_index=choice.index,
+            inner_content=chunk,
+            ai_model_id=self.ai_model_id,
+            metadata=choice_metadata,
+            text=text or "",
+        )
+
+    def _get_metadata_from_text_response(
+        self, response: TextCompletion | ChatCompletion | ChatCompletionChunk
+    ) -> dict[str, Any]:
+        """Get metadata from a response."""
+        ret = {
+            "id": response.id,
+            "created": response.created,
+            "system_fingerprint": response.system_fingerprint,
+        }
+        if response.usage is not None:
+            ret["usage"] = CompletionUsage.from_openai(response.usage)
+        return ret
+
+    def _get_metadata_from_text_choice(
+        self, choice: TextCompletionChoice | ChatCompletionChoice | ChatCompletionChunkChoice
+    ) -> dict[str, Any]:
+        """Get metadata from a completion choice."""
+        return {
+            "logprobs": getattr(choice, "logprobs", None),
+        }

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding.py

@@ -0,0 +1,90 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding_base import OpenAITextEmbeddingBase
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.utils.feature_stage_decorator import experimental
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+T_ = TypeVar("T_", bound="OpenAITextEmbedding")
+
+
+@experimental
+class OpenAITextEmbedding(OpenAIConfigBase, OpenAITextEmbeddingBase):
+    """OpenAI Text Embedding class."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """Initializes a new instance of the OpenAITextCompletion class.
+
+        Args:
+            ai_model_id (str): OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id (str | None): Service ID tied to the execution settings.
+            api_key (str | None): The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id (str | None): The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers (Mapping[str,str] | None): The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional)
+            env_file_path (str | None): Use the environment settings file as
+                a fallback to environment variables. (Optional)
+            env_file_encoding (str | None): The encoding of the environment settings file. (Optional)
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                embedding_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.embedding_model_id:
+            raise ServiceInitializationError("The OpenAI embedding model ID is required.")
+        super().__init__(
+            ai_model_id=openai_settings.embedding_model_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            ai_model_type=OpenAIModelTypes.EMBEDDING,
+            org_id=openai_settings.org_id,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        return cls(
+            ai_model_id=settings.get("ai_model_id"),
+            api_key=settings.get("api_key"),
+            org_id=settings.get("org_id"),
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers", {}),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_embedding_base.py

@@ -0,0 +1,77 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import sys
+from typing import TYPE_CHECKING, Any
+
+from numpy import array, ndarray
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
+    OpenAIEmbeddingPromptExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.utils.feature_stage_decorator import experimental
+
+if TYPE_CHECKING:
+    from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+
+
+@experimental
+class OpenAITextEmbeddingBase(OpenAIHandler, EmbeddingGeneratorBase):
+    """Base class for OpenAI text embedding services."""
+
+    @override
+    async def generate_embeddings(
+        self,
+        texts: list[str],
+        settings: "PromptExecutionSettings | None" = None,
+        batch_size: int | None = None,
+        **kwargs: Any,
+    ) -> ndarray:
+        raw_embeddings = await self.generate_raw_embeddings(texts, settings, batch_size, **kwargs)
+        return array(raw_embeddings)
+
+    @override
+    async def generate_raw_embeddings(
+        self,
+        texts: list[str],
+        settings: "PromptExecutionSettings | None" = None,
+        batch_size: int | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        """Returns embeddings for the given texts in the unedited format.
+
+        Args:
+            texts (List[str]): The texts to generate embeddings for.
+            settings (PromptExecutionSettings): The settings to use for the request.
+            batch_size (int): The batch size to use for the request.
+            kwargs (Dict[str, Any]): Additional arguments to pass to the request.
+        """
+        if not settings:
+            settings = OpenAIEmbeddingPromptExecutionSettings(ai_model_id=self.ai_model_id)
+        else:
+            if not isinstance(settings, OpenAIEmbeddingPromptExecutionSettings):
+                settings = self.get_prompt_execution_settings_from_settings(settings)
+        assert isinstance(settings, OpenAIEmbeddingPromptExecutionSettings)  # nosec
+        if settings.ai_model_id is None:
+            settings.ai_model_id = self.ai_model_id
+        for key, value in kwargs.items():
+            setattr(settings, key, value)
+        raw_embeddings = []
+        batch_size = batch_size or len(texts)
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i : i + batch_size]
+            settings.input = batch
+            raw_embedding = await self._send_request(settings=settings)
+            assert isinstance(raw_embedding, list)  # nosec
+            raw_embeddings.extend(raw_embedding)
+        return raw_embeddings
+
+    def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
+        """Get the request settings class."""
+        return OpenAIEmbeddingPromptExecutionSettings

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_audio.py

@@ -0,0 +1,85 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio_base import OpenAITextToAudioBase
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="OpenAITextToAudio")
+
+
+class OpenAITextToAudio(OpenAIConfigBase, OpenAITextToAudioBase):
+    """OpenAI Text to Image service."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """Initializes a new instance of the OpenAITextToAudio class.
+
+        Args:
+            ai_model_id: OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id: Service ID tied to the execution settings.
+            api_key: The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id: The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as
+                a fallback to environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                text_to_audio_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.text_to_audio_model_id:
+            raise ServiceInitializationError("The OpenAI text to audio model ID is required.")
+        super().__init__(
+            ai_model_id=openai_settings.text_to_audio_model_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            ai_model_type=OpenAIModelTypes.TEXT_TO_AUDIO,
+            org_id=openai_settings.org_id,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        return cls(
+            ai_model_id=settings.get("ai_model_id"),
+            api_key=settings.get("api_key"),
+            org_id=settings.get("org_id"),
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers", {}),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_audio_base.py

@@ -0,0 +1,57 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import sys
+from typing import Any
+
+from openai import _legacy_response
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_audio_execution_settings import (
+    OpenAITextToAudioExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.connectors.ai.text_to_audio_client_base import TextToAudioClientBase
+from semantic_kernel.contents.audio_content import AudioContent
+
+
+class OpenAITextToAudioBase(OpenAIHandler, TextToAudioClientBase):
+    """OpenAI text to audio client base class."""
+
+    @override
+    async def get_audio_contents(
+        self,
+        text: str,
+        settings: PromptExecutionSettings | None = None,
+        **kwargs: Any,
+    ) -> list[AudioContent]:
+        if not settings:
+            settings = OpenAITextToAudioExecutionSettings(ai_model_id=self.ai_model_id)
+        else:
+            if not isinstance(settings, OpenAITextToAudioExecutionSettings):
+                settings = self.get_prompt_execution_settings_from_settings(settings)
+
+        assert isinstance(settings, OpenAITextToAudioExecutionSettings)  # nosec
+
+        if settings.ai_model_id is None:
+            settings.ai_model_id = self.ai_model_id
+        settings.input = text
+
+        response = await self._send_request(settings)
+        assert isinstance(response, _legacy_response.HttpxBinaryResponseContent)  # nosec
+
+        return [
+            AudioContent(
+                ai_model_id=settings.ai_model_id,
+                data=response.read(),
+                data_format="base64",
+            )
+        ]
+
+    def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
+        """Get the request settings class."""
+        return OpenAITextToAudioExecutionSettings

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image.py

@@ -0,0 +1,85 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from openai import AsyncOpenAI
+from pydantic import ValidationError
+
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image_base import OpenAITextToImageBase
+from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+
+T_ = TypeVar("T_", bound="OpenAITextToImage")
+
+
+class OpenAITextToImage(OpenAIConfigBase, OpenAITextToImageBase):
+    """OpenAI Text to Image service."""
+
+    def __init__(
+        self,
+        ai_model_id: str | None = None,
+        api_key: str | None = None,
+        org_id: str | None = None,
+        service_id: str | None = None,
+        default_headers: Mapping[str, str] | None = None,
+        async_client: AsyncOpenAI | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """Initializes a new instance of the OpenAITextToImage class.
+
+        Args:
+            ai_model_id: OpenAI model name, see
+                https://platform.openai.com/docs/models
+            service_id: Service ID tied to the execution settings.
+            api_key: The optional API key to use. If provided will override,
+                the env vars or .env file value.
+            org_id: The optional org ID to use. If provided will override,
+                the env vars or .env file value.
+            default_headers: The default headers mapping of string keys to
+                string values for HTTP requests. (Optional)
+            async_client: An existing client to use. (Optional)
+            env_file_path: Use the environment settings file as
+                a fallback to environment variables. (Optional)
+            env_file_encoding: The encoding of the environment settings file. (Optional)
+        """
+        try:
+            openai_settings = OpenAISettings(
+                api_key=api_key,
+                org_id=org_id,
+                text_to_image_model_id=ai_model_id,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex
+        if not openai_settings.text_to_image_model_id:
+            raise ServiceInitializationError("The OpenAI text to image model ID is required.")
+        super().__init__(
+            ai_model_id=openai_settings.text_to_image_model_id,
+            api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None,
+            ai_model_type=OpenAIModelTypes.TEXT_TO_IMAGE,
+            org_id=openai_settings.org_id,
+            service_id=service_id,
+            default_headers=default_headers,
+            client=async_client,
+        )
+
+    @classmethod
+    def from_dict(cls: type[T_], settings: dict[str, Any]) -> T_:
+        """Initialize an Open AI service from a dictionary of settings.
+
+        Args:
+            settings: A dictionary of settings for the service.
+        """
+        return cls(
+            ai_model_id=settings.get("ai_model_id"),
+            api_key=settings.get("api_key"),
+            org_id=settings.get("org_id"),
+            service_id=settings.get("service_id"),
+            default_headers=settings.get("default_headers", {}),
+            env_file_path=settings.get("env_file_path"),
+        )

python/semantic_kernel/connectors/ai/open_ai/services/open_ai_text_to_image_base.py

@@ -0,0 +1,249 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from pathlib import Path
+from typing import IO, Any
+from warnings import warn
+
+from openai._types import FileTypes, Omit, omit
+from openai.types.images_response import ImagesResponse
+
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_image_execution_settings import (
+    ImageSize,
+    OpenAITextToImageExecutionSettings,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
+from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
+from semantic_kernel.connectors.ai.text_to_image_client_base import TextToImageClientBase
+from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError, ServiceResponseException
+
+
+class OpenAITextToImageBase(OpenAIHandler, TextToImageClientBase):
+    """OpenAI text to image client."""
+
+    async def generate_image(
+        self,
+        description: str,
+        width: int | None = None,
+        height: int | None = None,
+        settings: PromptExecutionSettings | None = None,
+        **kwargs: Any,
+    ) -> bytes | str:
+        """Generate image from text.
+
+        Args:
+            description: Description of the image.
+            width: Deprecated, use settings instead.
+            height: Deprecated, use settings instead.
+            settings: Execution settings for the prompt.
+            kwargs: Additional arguments, check the openai images.generate documentation for the supported arguments.
+
+        Returns:
+            bytes | str: Image bytes or image URL.
+        """
+        warn("generate_image is deprecated. Use generate_images.", DeprecationWarning, stacklevel=2)
+        if not settings:
+            settings = OpenAITextToImageExecutionSettings(**kwargs)
+        if not isinstance(settings, OpenAITextToImageExecutionSettings):
+            settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
+        if width:
+            warn("The 'width' argument is deprecated. Use 'settings.size' instead.", DeprecationWarning)
+            if settings.size and not settings.size.width:
+                settings.size.width = width
+        if height:
+            warn("The 'height' argument is deprecated. Use 'settings.size' instead.", DeprecationWarning)
+            if settings.size and not settings.size.height:
+                settings.size.height = height
+        if not settings.size and width and height:
+            settings.size = ImageSize(width=width, height=height)
+
+        if not settings.prompt:
+            settings.prompt = description
+
+        if not settings.prompt:
+            raise ServiceInvalidRequestError("Prompt is required.")
+
+        if not settings.ai_model_id:
+            settings.ai_model_id = self.ai_model_id
+
+        response = await self._send_request(settings)
+
+        assert isinstance(response, ImagesResponse)  # nosec
+        if not response.data or not (response.data[0].url or response.data[0].b64_json):
+            raise ServiceResponseException("Failed to generate image.")
+
+        return response.data[0].url or response.data[0].b64_json  # type: ignore[return-value]
+
+    async def generate_images(
+        self,
+        prompt: str,
+        settings: PromptExecutionSettings | None = None,
+        **kwargs: Any,
+    ) -> list[str]:
+        """Generate one or more images from text. Returns URLs or base64-encoded images.
+
+        Args:
+            prompt: Description of the image(s) to generate.
+            settings: Execution settings for the prompt.
+            kwargs: Additional arguments, check the openai images.generate documentation for the supported arguments.
+
+        Returns:
+            list[str]: Image URLs or base64-encoded images.
+
+        Example:
+            Generate images and save them as PNG files:
+
+            ```python
+            from semantic_kernel.connectors.ai.open_ai import AzureTextToImage
+            import base64, os
+
+            service = AzureTextToImage(
+                service_id="image1",
+                deployment_name="gpt-image-1",
+                endpoint="https://your-endpoint.cognitiveservices.azure.com",
+                api_key="your-api-key",
+                api_version="2025-04-01-preview",
+            )
+            settings = service.get_prompt_execution_settings_class()(service_id="image1")
+            settings.n = 3
+            images_b64 = await service.generate_images("A cute cat wearing a whimsical striped hat", settings=settings)
+            ```
+        """
+        if not settings:
+            settings = OpenAITextToImageExecutionSettings(**kwargs)
+        if not isinstance(settings, OpenAITextToImageExecutionSettings):
+            settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
+        if prompt:
+            settings.prompt = prompt
+
+        if not settings.prompt:
+            raise ServiceInvalidRequestError("Prompt is required.")
+
+        if not settings.ai_model_id:
+            settings.ai_model_id = self.ai_model_id
+
+        response = await self._send_request(settings)
+
+        assert isinstance(response, ImagesResponse)  # nosec
+        if not response.data or not isinstance(response.data, list) or len(response.data) == 0:
+            raise ServiceResponseException("Failed to generate image.")
+
+        results: list[str] = []
+        for image in response.data:
+            url: str | None = getattr(image, "url", None)
+            b64_json: str | None = getattr(image, "b64_json", None)
+            if url:
+                results.append(url)
+            elif b64_json:
+                results.append(b64_json)
+            else:
+                continue
+
+        if len(results) == 0:
+            raise ServiceResponseException("No valid image data found in response.")
+        return results
+
+    async def edit_image(
+        self,
+        prompt: str,
+        image_paths: list[str] | None = None,
+        image_files: list[IO[bytes]] | None = None,
+        mask_path: str | None = None,
+        mask_file: IO[bytes] | None = None,
+        settings: PromptExecutionSettings | None = None,
+        **kwargs: Any,
+    ) -> list[str]:
+        """Edit images using the OpenAI image edit API.
+
+        Args:
+            prompt: Instructional prompt for image editing.
+            image_paths: List of image file paths to edit.
+            image_files: List of file-like objects (opened in binary mode) to edit.
+            mask_path: Optional mask image file path.
+            mask_file: Optional mask image file-like object (opened in binary mode).
+            settings: Optional execution settings. If not provided, will be constructed from kwargs.
+            kwargs: Additional API parameters.
+
+        Returns:
+            list[str]: List of edited image URLs or base64-encoded strings.
+
+        Example:
+            Edit images from file path and save results:
+
+            ```python
+            from semantic_kernel.connectors.ai.open_ai import AzureTextToImage
+            import base64, os
+
+            service = AzureTextToImage(
+                service_id="image1",
+                deployment_name="gpt-image-1",
+                endpoint="https://your-endpoint.cognitiveservices.azure.com",
+                api_key="your-api-key",
+                api_version="2025-04-01-preview",
+            )
+            file_paths = ["./new_images/img_1.png", "./new_images/img_2.png"]
+            settings = service.get_prompt_execution_settings_class()(service_id="image1")
+            settings.n = 2
+            results = await service.edit_image(
+                prompt="Make the cat wear a wizard hat",
+                image_paths=file_paths,
+                settings=settings,
+            )
+            ```
+
+            Edit images from file object:
+
+            ```python
+            with open("./new_images/img_1.png", "rb") as f:
+                results = await service.edit_image(
+                    prompt="Make the cat wear a wizard hat",
+                    image_files=[f],
+                )
+            ```
+        """
+        if not settings:
+            settings = OpenAITextToImageExecutionSettings(**kwargs)
+        if not isinstance(settings, OpenAITextToImageExecutionSettings):
+            settings = OpenAITextToImageExecutionSettings.from_prompt_execution_settings(settings)
+        settings.prompt = prompt
+
+        if not settings.prompt:
+            raise ServiceInvalidRequestError("Prompt is required.")
+        if (image_paths is None and image_files is None) or (image_paths is not None and image_files is not None):
+            raise ServiceInvalidRequestError("Provide either 'image_paths' or 'image_files', and only one.")
+
+        images: list[FileTypes] = []
+        if image_paths is not None:
+            images = [Path(p) for p in image_paths]
+        elif image_files is not None:
+            images = list(image_files)
+
+        mask: FileTypes | Omit = omit
+        if mask_path is not None:
+            mask = Path(mask_path)
+        elif mask_file is not None:
+            mask = mask_file
+
+        response: ImagesResponse = await self._send_image_edit_request(
+            image=images,
+            mask=mask,
+            settings=settings,
+        )
+
+        if not response or not response.data or not isinstance(response.data, list):
+            raise ServiceResponseException("Failed to edit image.")
+
+        results: list[str] = []
+        for img in response.data:
+            b64_json: str | None = getattr(img, "b64_json", None)
+            url: str | None = getattr(img, "url", None)
+            if b64_json:
+                results.append(b64_json)
+            elif url:
+                results.append(url)
+        if not results:
+            raise ServiceResponseException("No valid image data found in response.")
+        return results
+
+    def get_prompt_execution_settings_class(self) -> type[PromptExecutionSettings]:
+        """Get the request settings class."""
+        return OpenAITextToImageExecutionSettings

python/semantic_kernel/connectors/ai/open_ai/settings/azure_open_ai_settings.py

@@ -0,0 +1,135 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import ClassVar
+
+from azure.core.credentials import TokenCredential
+from pydantic import SecretStr
+from pydantic_core import Url
+
+from semantic_kernel.connectors.ai.open_ai.const import DEFAULT_AZURE_API_VERSION
+from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.kernel_pydantic import HttpsUrl, KernelBaseSettings
+from semantic_kernel.utils.authentication.entra_id_authentication import get_entra_auth_token
+
+
+class AzureOpenAISettings(KernelBaseSettings):
+    """AzureOpenAI model settings.
+
+    The settings are first loaded from environment variables with the prefix 'AZURE_OPENAI_'.
+    If the environment variables are not found, the settings can be loaded from a .env file
+    with the encoding 'utf-8'. If the settings are not found in the .env file, the settings
+    are ignored; however, validation will fail alerting that the settings are missing.
+
+    Optional settings for prefix 'AZURE_OPENAI_' are:
+    - chat_deployment_name: str - The name of the Azure Chat deployment. This value
+                will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_CHAT_DEPLOYMENT_NAME)
+    - responses_deployment_name: str - The name of the Azure Responses deployment. This value
+                will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME)
+    - text_deployment_name: str - The name of the Azure Text deployment. This value
+                will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_TEXT_DEPLOYMENT_NAME)
+    - embedding_deployment_name: str - The name of the Azure Embedding deployment. This value
+                will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME)
+    - text_to_image_deployment_name: str - The name of the Azure Text to Image deployment. This
+                value will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_TEXT_TO_IMAGE_DEPLOYMENT_NAME)
+    - audio_to_text_deployment_name: str - The name of the Azure Audio to Text deployment. This
+                value will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_AUDIO_TO_TEXT_DEPLOYMENT_NAME)
+    - text_to_audio_deployment_name: str - The name of the Azure Text to Audio deployment. This
+                value will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_TEXT_TO_AUDIO_DEPLOYMENT_NAME)
+    - realtime_deployment_name: str - The name of the Azure Realtime deployment. This value
+                will correspond to the custom name you chose for your deployment
+                when you deployed a model. This value can be found under
+                Resource Management > Deployments in the Azure portal or, alternatively,
+                under Management > Deployments in Azure AI Foundry.
+                (Env var AZURE_OPENAI_REALTIME_DEPLOYMENT_NAME)
+    - api_key: SecretStr - The API key for the Azure deployment. This value can be
+                found in the Keys & Endpoint section when examining your resource in
+                the Azure portal. You can use either KEY1 or KEY2.
+                (Env var AZURE_OPENAI_API_KEY)
+    - base_url: HttpsUrl | None - base_url: The url of the Azure deployment. This value
+                can be found in the Keys & Endpoint section when examining
+                your resource from the Azure portal, the base_url consists of the endpoint,
+                followed by /openai/deployments/{deployment_name}/,
+                use endpoint if you only want to supply the endpoint.
+                (Env var AZURE_OPENAI_BASE_URL)
+    - endpoint: HttpsUrl - The endpoint of the Azure deployment. This value
+                can be found in the Keys & Endpoint section when examining
+                your resource from the Azure portal, the endpoint should end in openai.azure.com.
+                If both base_url and endpoint are supplied, base_url will be used.
+                (Env var AZURE_OPENAI_ENDPOINT)
+    - api_version: str | None - The API version to use. The default value is "2024-10-21".
+                (Env var AZURE_OPENAI_API_VERSION)
+    - token_endpoint: str - The token endpoint to use to retrieve the authentication token.
+                The default value is "https://cognitiveservices.azure.com/.default".
+                (Env var AZURE_OPENAI_TOKEN_ENDPOINT)
+    """
+
+    env_prefix: ClassVar[str] = "AZURE_OPENAI_"
+
+    chat_deployment_name: str | None = None
+    responses_deployment_name: str | None = None
+    text_deployment_name: str | None = None
+    embedding_deployment_name: str | None = None
+    text_to_image_deployment_name: str | None = None
+    audio_to_text_deployment_name: str | None = None
+    text_to_audio_deployment_name: str | None = None
+    realtime_deployment_name: str | None = None
+    endpoint: HttpsUrl | None = None
+    base_url: Url | None = None
+    api_key: SecretStr | None = None
+    api_version: str = DEFAULT_AZURE_API_VERSION
+    token_endpoint: str = "https://cognitiveservices.azure.com/.default"
+
+    def get_azure_openai_auth_token(
+        self, credential: TokenCredential | None = None, token_endpoint: str | None = None
+    ) -> str | None:
+        """Retrieve a Microsoft Entra Auth Token for a given token endpoint for the use with Azure OpenAI.
+
+        The required role for the token is `Cognitive Services OpenAI Contributor`.
+        The token endpoint may be specified as an environment variable, via the .env
+        file or as an argument. If the token endpoint is not provided, the default is None.
+        The `token_endpoint` argument takes precedence over the `token_endpoint` attribute.
+
+        Args:
+            credential: The credential to use for authentication.
+            token_endpoint: The token endpoint to use. Defaults to `https://cognitiveservices.azure.com/.default`.
+
+        Returns:
+            The Azure token or None if the token could not be retrieved.
+
+        Raises:
+            ServiceInitializationError: If the token endpoint is not provided.
+        """
+        endpoint_to_use = token_endpoint or self.token_endpoint
+        if endpoint_to_use is None:
+            raise ServiceInitializationError("Please provide a token endpoint to retrieve the authentication token.")
+        if credential is None:
+            raise ServiceInitializationError("Please provide a credential to retrieve the authentication token.")
+        return get_entra_auth_token(credential, endpoint_to_use)

python/semantic_kernel/connectors/ai/open_ai/settings/open_ai_settings.py

@@ -0,0 +1,54 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import ClassVar
+
+from pydantic import SecretStr
+
+from semantic_kernel.kernel_pydantic import KernelBaseSettings
+
+
+class OpenAISettings(KernelBaseSettings):
+    """OpenAI model settings.
+
+    The settings are first loaded from environment variables with the prefix 'OPENAI_'.
+    If the environment variables are not found, the settings can be loaded from a .env file with the
+    encoding 'utf-8'. If the settings are not found in the .env file, the settings are ignored;
+    however, validation will fail alerting that the settings are missing.
+
+    Optional settings for prefix 'OPENAI_' are:
+    - api_key: SecretStr - OpenAI API key, see https://platform.openai.com/account/api-keys
+        (Env var OPENAI_API_KEY)
+    - org_id: str | None - This is usually optional unless your account belongs to multiple organizations.
+        (Env var OPENAI_ORG_ID)
+    - chat_model_id: str | None - The OpenAI chat model ID to use, for example, gpt-3.5-turbo or gpt-4.
+        (Env var OPENAI_CHAT_MODEL_ID)
+    - responses_model_id: str | None - The OpenAI responses model ID to use, for example, gpt-4o or o1.
+        (Env var OPENAI_RESPONSES_MODEL_ID)
+    - text_model_id: str | None - The OpenAI text model ID to use, for example, gpt-3.5-turbo-instruct.
+        (Env var OPENAI_TEXT_MODEL_ID)
+    - embedding_model_id: str | None - The OpenAI embedding model ID to use, for example, text-embedding-ada-002.
+        (Env var OPENAI_EMBEDDING_MODEL_ID)
+    - text_to_image_model_id: str | None - The OpenAI text to image model ID to use, for example, dall-e-3.
+        (Env var OPENAI_TEXT_TO_IMAGE_MODEL_ID)
+    - audio_to_text_model_id: str | None - The OpenAI audio to text model ID to use, for example, whisper-1.
+        (Env var OPENAI_AUDIO_TO_TEXT_MODEL_ID)
+    - text_to_audio_model_id: str | None - The OpenAI text to audio model ID to use, for example, jukebox-1.
+        (Env var OPENAI_TEXT_TO_AUDIO_MODEL_ID)
+    - realtime_model_id: str | None - The OpenAI realtime model ID to use,
+        for example, gpt-realtime, gpt-realtime-mini, or gpt-audio-mini.
+        (Env var OPENAI_REALTIME_MODEL_ID)
+    - env_file_path: str | None - if provided, the .env settings are read from this file path location
+    """
+
+    env_prefix: ClassVar[str] = "OPENAI_"
+
+    api_key: SecretStr | None = None
+    org_id: str | None = None
+    chat_model_id: str | None = None
+    responses_model_id: str | None = None
+    text_model_id: str | None = None
+    embedding_model_id: str | None = None
+    text_to_image_model_id: str | None = None
+    audio_to_text_model_id: str | None = None
+    text_to_audio_model_id: str | None = None
+    realtime_model_id: str | None = None