chore: import upstream snapshot with attribution

haystack/components/generators/chat/__init__.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "openai": ["OpenAIChatGenerator"],
+    "openai_responses": ["OpenAIResponsesChatGenerator"],
+    "azure": ["AzureOpenAIChatGenerator"],
+    "azure_responses": ["AzureOpenAIResponsesChatGenerator"],
+    "fallback": ["FallbackChatGenerator"],
+    "llm": ["LLM"],
+    "mock": ["MockChatGenerator"],
+}
+
+if TYPE_CHECKING:
+    from .azure import AzureOpenAIChatGenerator as AzureOpenAIChatGenerator
+    from .azure_responses import AzureOpenAIResponsesChatGenerator as AzureOpenAIResponsesChatGenerator
+    from .fallback import FallbackChatGenerator as FallbackChatGenerator
+    from .llm import LLM as LLM
+    from .mock import MockChatGenerator as MockChatGenerator
+    from .openai import OpenAIChatGenerator as OpenAIChatGenerator
+    from .openai_responses import OpenAIResponsesChatGenerator as OpenAIResponsesChatGenerator
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/generators/chat/azure.py

@@ -0,0 +1,369 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any, ClassVar
+
+from openai.lib._pydantic import to_strict_json_schema
+from openai.lib.azure import AsyncAzureADTokenProvider, AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.dataclasses.streaming_chunk import StreamingCallbackT
+from haystack.tools import (
+    ToolsType,
+    _check_duplicate_tool_names,
+    deserialize_tools_or_toolset_inplace,
+    flatten_tools_or_toolsets,
+    serialize_tools_or_toolset,
+    warm_up_tools,
+)
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+
+@component
+class AzureOpenAIChatGenerator(OpenAIChatGenerator):
+    """
+    Generates text using OpenAI's models on Azure.
+
+    It works with the gpt-4 - type models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.ChatCompletion.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.generators.chat import AzureOpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+    from haystack.utils import Secret
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = AzureOpenAIChatGenerator(
+        azure_endpoint="<Your Azure endpoint e.g. `https://your-company.azure.openai.com/>",
+        api_key=Secret.from_token("<your-api-key>"),
+        azure_deployment="<this is a model name, e.g. gpt-4.1-mini>")
+    response = client.run(messages)
+    print(response)
+    ```
+
+    ```
+    {'replies':
+        [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
+        "Natural Language Processing (NLP) is a branch of artificial intelligence that focuses on
+         enabling computers to understand, interpret, and generate human language in a way that is useful.")],
+         _name=None,
+         _meta={'model': 'gpt-4.1-mini', 'index': 0, 'finish_reason': 'stop',
+         'usage': {'prompt_tokens': 15, 'completion_tokens': 36, 'total_tokens': 51}})]
+    }
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5.4",
+        "gpt-5.4-pro",
+        "gpt-5.3-codex",
+        "gpt-5.2",
+        "gpt-5.2-codex",
+        "gpt-5.2-chat",
+        "gpt-5.1",
+        "gpt-5.1-chat",
+        "gpt-5.1-codex",
+        "gpt-5.1-codex-mini",
+        "gpt-5",
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5-chat",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-4.1-nano",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4o-audio-preview",
+        "gpt-realtime-1.5",
+        "gpt-audio-1.5",
+        "o1",
+        "o1-mini",
+        "o3",
+        "o3-mini",
+        "o4-mini",
+        "codex-mini",
+        "gpt-4",
+        "gpt-35-turbo",
+        "gpt-oss-120b",
+        "computer-use-preview",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure
+    for the full list."""
+
+    # ruff: noqa: PLR0913
+    def __init__(
+        self,
+        azure_endpoint: str | Secret | None = None,
+        api_version: str | Secret | None = "2024-12-01-preview",
+        azure_deployment: str | None = "gpt-4.1-mini",
+        api_key: Secret | None = Secret.from_env_var("AZURE_OPENAI_API_KEY", strict=False),
+        azure_ad_token: Secret | None = Secret.from_env_var("AZURE_OPENAI_AD_TOKEN", strict=False),
+        organization: str | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        default_headers: dict[str, str] | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        *,
+        azure_ad_token_provider: AzureADTokenProvider | AsyncAzureADTokenProvider | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initialize the Azure OpenAI Chat Generator component.
+
+        :param azure_endpoint: The endpoint of the deployed model, for example `"https://example-resource.azure.openai.com/"`.
+            Can also be a [Secret](https://docs.haystack.deepset.ai/docs/secret-management), for example
+            `Secret.from_env_var("AZURE_OPENAI_ENDPOINT")`, to resolve the value from an environment variable at
+            runtime. This is useful to switch endpoints between environments (e.g. dev and prod) without changing the
+            serialized pipeline.
+        :param api_version: The version of the API to use. Defaults to 2024-12-01-preview.
+            Can also be a [Secret](https://docs.haystack.deepset.ai/docs/secret-management), for example
+            `Secret.from_env_var("AZURE_OPENAI_API_VERSION")`, to resolve the value from an environment variable at
+            runtime.
+        :param azure_deployment: The deployment of the model, usually the model name.
+        :param api_key: The API key to use for authentication.
+        :param azure_ad_token: [Azure Active Directory token](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id).
+        :param organization: Your organization ID, defaults to `None`. For help, see
+        [Setting up your organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param streaming_callback: A callback function called when a new token is received from the stream.
+            It accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param timeout: Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent directly to
+            the OpenAI endpoint. For details, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+            Some of the supported parameters:
+            - `max_completion_tokens`: An upper bound for the number of tokens that can be generated for a completion,
+                including visible output tokens and reasoning tokens.
+            - `temperature`: The sampling temperature to use. Higher values mean the model takes more risks.
+                Try 0.9 for more creative applications and 0 (argmax sampling) for ones with a well-defined answer.
+            - `top_p`: Nucleus sampling is an alternative to sampling with temperature, where the model considers
+                tokens with a top_p probability mass. For example, 0.1 means only the tokens comprising
+                the top 10% probability mass are considered.
+            - `n`: The number of completions to generate for each prompt. For example, with 3 prompts and n=2,
+                the LLM will generate two completions per prompt, resulting in 6 completions total.
+            - `stop`: One or more sequences after which the LLM should stop generating tokens.
+            - `presence_penalty`: The penalty applied if a token is already present.
+                Higher values make the model less likely to repeat the token.
+            - `frequency_penalty`: Penalty applied if a token has already been generated.
+                Higher values make the model less likely to repeat the token.
+            - `logit_bias`: Adds a logit bias to specific tokens. The keys of the dictionary are tokens, and the
+                values are the bias to add to that token.
+            - `response_format`: A JSON schema or a Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+                Notes:
+                - This parameter accepts Pydantic models and JSON schemas for latest models starting from GPT-4o.
+                  Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                  For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+                - For structured outputs with streaming,
+                  the `response_format` must be a JSON schema and not a Pydantic model.
+        :param default_headers: Default headers to use for the AzureOpenAI client.
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param azure_ad_token_provider: A function that returns an Azure Active Directory token, will be invoked on
+            every request.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        # We intentionally do not call super().__init__ here because we only need to instantiate the client to interact
+        # with the API.
+
+        # Why is this here?
+        # AzureOpenAI init is forcing us to use an init method that takes either base_url or azure_endpoint as not
+        # None init parameters. This way we accommodate the use case where env var AZURE_OPENAI_ENDPOINT is set instead
+        # of passing it as a parameter.
+        azure_endpoint = azure_endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")
+        # `azure_endpoint` accepts either a plain string or a `Secret`. We keep the original value on the instance for
+        # serialization and resolve it to a string only to validate that an endpoint was provided.
+        resolved_azure_endpoint = (
+            azure_endpoint.resolve_value() if isinstance(azure_endpoint, Secret) else azure_endpoint
+        )
+        if not resolved_azure_endpoint:
+            raise ValueError("Please provide an Azure endpoint or set the environment variable AZURE_OPENAI_ENDPOINT.")
+
+        if api_key is None and azure_ad_token is None:
+            raise ValueError("Please provide an API key or an Azure Active Directory token.")
+
+        # The check above makes mypy incorrectly infer that api_key is never None,
+        # which propagates the incorrect type.
+        self.api_key = api_key  # type: ignore
+        self.azure_ad_token = azure_ad_token
+        self.generation_kwargs = generation_kwargs or {}
+        self.streaming_callback = streaming_callback
+        self.api_version = api_version
+        self.azure_endpoint = azure_endpoint
+        self.azure_deployment = azure_deployment
+        self.organization = organization
+        self.model = azure_deployment or "gpt-4.1-mini"
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.default_headers = default_headers or {}
+        self.azure_ad_token_provider = azure_ad_token_provider
+        self.http_client_kwargs = http_client_kwargs
+        _check_duplicate_tool_names(flatten_tools_or_toolsets(tools))
+        self.tools = tools
+        self.tools_strict = tools_strict
+
+        self.client: AzureOpenAI | None = None
+        self.async_client: AsyncAzureOpenAI | None = None
+        self._tools_warmed_up = False
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        resolved_azure_endpoint = (
+            self.azure_endpoint.resolve_value() if isinstance(self.azure_endpoint, Secret) else self.azure_endpoint
+        )
+        resolved_api_version = (
+            self.api_version.resolve_value() if isinstance(self.api_version, Secret) else self.api_version
+        )
+        return {
+            "api_version": resolved_api_version,
+            "azure_endpoint": resolved_azure_endpoint,
+            "azure_deployment": self.azure_deployment,
+            "api_key": self.api_key.resolve_value() if self.api_key is not None else None,
+            "azure_ad_token": self.azure_ad_token.resolve_value() if self.azure_ad_token is not None else None,
+            "organization": self.organization,
+            "timeout": timeout,
+            "max_retries": max_retries,
+            "default_headers": self.default_headers,
+            "azure_ad_token_provider": self.azure_ad_token_provider,
+        }
+
+    def _warm_up_tools(self) -> None:
+        if not self._tools_warmed_up:
+            warm_up_tools(self.tools)
+            self._tools_warmed_up = True
+
+    def warm_up(self) -> None:
+        """
+        Warm up the tools and initialize the synchronous Azure OpenAI client.
+        """
+        self._warm_up_tools()
+        if self.client is None:
+            self.client = AzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Warm up the tools and initialize the asynchronous Azure OpenAI client on the serving event loop.
+        """
+        self._warm_up_tools()
+        if self.async_client is None:
+            self.async_client = AsyncAzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous Azure OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous Azure OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        azure_ad_token_provider_name = None
+        if self.azure_ad_token_provider:
+            azure_ad_token_provider_name = serialize_callable(self.azure_ad_token_provider)
+        # If the response format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        generation_kwargs = self.generation_kwargs.copy()
+        response_format = generation_kwargs.get("response_format")
+        if response_format and issubclass(response_format, BaseModel):
+            json_schema = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": response_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(response_format),
+                },
+            }
+            generation_kwargs["response_format"] = json_schema
+        return default_to_dict(
+            self,
+            azure_endpoint=self.azure_endpoint.to_dict()
+            if isinstance(self.azure_endpoint, Secret)
+            else self.azure_endpoint,
+            azure_deployment=self.azure_deployment,
+            organization=self.organization,
+            api_version=self.api_version.to_dict() if isinstance(self.api_version, Secret) else self.api_version,
+            streaming_callback=callback_name,
+            generation_kwargs=generation_kwargs,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            api_key=self.api_key,
+            azure_ad_token=self.azure_ad_token,
+            default_headers=self.default_headers,
+            tools=serialize_tools_or_toolset(self.tools),
+            tools_strict=self.tools_strict,
+            azure_ad_token_provider=azure_ad_token_provider_name,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        serialized_azure_ad_token_provider = init_params.get("azure_ad_token_provider")
+        if serialized_azure_ad_token_provider:
+            data["init_parameters"]["azure_ad_token_provider"] = deserialize_callable(
+                serialized_azure_ad_token_provider
+            )
+        return default_from_dict(cls, data)

haystack/components/generators/chat/azure_responses.py

@@ -0,0 +1,272 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from collections.abc import Awaitable, Callable
+from typing import Any, ClassVar
+
+from openai.lib._pydantic import to_strict_json_schema
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.generators.chat import OpenAIResponsesChatGenerator
+from haystack.dataclasses.streaming_chunk import StreamingCallbackT
+from haystack.tools import ToolsType, deserialize_tools_or_toolset_inplace, serialize_tools_or_toolset
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+
+
+@component
+class AzureOpenAIResponsesChatGenerator(OpenAIResponsesChatGenerator):
+    """
+    Completes chats using OpenAI's Responses API on Azure.
+
+    It works with the gpt-5 and o-series models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.Responses.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/responses).
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.generators.chat import AzureOpenAIResponsesChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = AzureOpenAIResponsesChatGenerator(
+        azure_endpoint="https://example-resource.azure.openai.com/",
+        generation_kwargs={"reasoning": {"effort": "low", "summary": "auto"}}
+    )
+    response = client.run(messages)
+    print(response)
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5.4-pro",
+        "gpt-5.4",
+        "gpt-5.3-chat",
+        "gpt-5.3-codex",
+        "gpt-5.2-codex",
+        "gpt-5.2",
+        "gpt-5.2-chat",
+        "gpt-5.1-codex-max",
+        "gpt-5.1",
+        "gpt-5.1-chat",
+        "gpt-5.1-codex",
+        "gpt-5.1-codex-mini",
+        "gpt-5-pro",
+        "gpt-5-codex",
+        "gpt-5",
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5-chat",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "computer-use-preview",
+        "gpt-4.1",
+        "gpt-4.1-nano",
+        "gpt-4.1-mini",
+        "gpt-image-1",
+        "gpt-image-1-mini",
+        "gpt-image-1.5",
+        "o1",
+        "o3-mini",
+        "o3",
+        "o4-mini",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses#model-support for the full list."""
+
+    # ruff: noqa: PLR0913
+    def __init__(
+        self,
+        *,
+        api_key: Secret | Callable[[], str] | Callable[[], Awaitable[str]] = Secret.from_env_var(
+            "AZURE_OPENAI_API_KEY", strict=False
+        ),
+        azure_endpoint: str | None = None,
+        azure_deployment: str = "gpt-5-mini",
+        streaming_callback: StreamingCallbackT | None = None,
+        organization: str | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initialize the AzureOpenAIResponsesChatGenerator component.
+
+        :param api_key: The API key to use for authentication. Can be:
+            - A `Secret` object containing the API key.
+            - A `Secret` object containing the [Azure Active Directory token](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id).
+            - A function that returns an Azure Active Directory token.
+        :param azure_endpoint: The endpoint of the deployed model, for example `"https://example-resource.azure.openai.com/"`.
+        :param azure_deployment: The deployment of the model, usually the model name.
+        :param organization: Your organization ID, defaults to `None`. For help, see
+        [Setting up your organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param streaming_callback: A callback function called when a new token is received from the stream.
+            It accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param timeout: Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent
+           directly to the OpenAI endpoint.
+           See OpenAI [documentation](https://platform.openai.com/docs/api-reference/responses) for
+            more details.
+            Some of the supported parameters:
+            - `temperature`: What sampling temperature to use. Higher values like 0.8 will make the output more random,
+                while lower values like 0.2 will make it more focused and deterministic.
+            - `top_p`: An alternative to sampling with temperature, called nucleus sampling, where the model
+                considers the results of the tokens with top_p probability mass. For example, 0.1 means only the tokens
+                comprising the top 10% probability mass are considered.
+            - `previous_response_id`: The ID of the previous response.
+                Use this to create multi-turn conversations.
+            - `text_format`: A Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+            - `text`: A JSON schema that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                Notes:
+                - Both JSON Schema and Pydantic models are supported for latest models starting from GPT-4o.
+                - If both are provided, `text_format` takes precedence and json schema passed to `text` is ignored.
+                - Currently, this component doesn't support streaming for structured outputs.
+                - Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                    For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+            - `reasoning`: A dictionary of parameters for reasoning. For example:
+                - `summary`: The summary of the reasoning.
+                - `effort`: The level of effort to put into the reasoning. Can be `low`, `medium` or `high`.
+                - `generate_summary`: Whether to generate a summary of the reasoning.
+                Note: OpenAI does not return the reasoning tokens, but we can view summary if its enabled.
+                For details, see the [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning).
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        azure_endpoint = azure_endpoint or os.getenv("AZURE_OPENAI_ENDPOINT")
+        if azure_endpoint is None:
+            raise ValueError(
+                "You must provide `azure_endpoint` or set the `AZURE_OPENAI_ENDPOINT` environment variable."
+            )
+        self._azure_endpoint = azure_endpoint
+        self._azure_deployment = azure_deployment
+        super(AzureOpenAIResponsesChatGenerator, self).__init__(  # noqa: UP008
+            api_key=api_key,  # type: ignore[arg-type]
+            model=self._azure_deployment,
+            streaming_callback=streaming_callback,
+            api_base_url=f"{self._azure_endpoint.rstrip('/')}/openai/v1",
+            organization=organization,
+            generation_kwargs=generation_kwargs,
+            timeout=timeout,
+            max_retries=max_retries,
+            tools=tools,
+            tools_strict=tools_strict,
+            http_client_kwargs=http_client_kwargs,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+
+        # API key can be a secret or a callable
+        serialized_api_key = (
+            serialize_callable(self.api_key)
+            if callable(self.api_key)
+            else self.api_key.to_dict()
+            if isinstance(self.api_key, Secret)
+            else None
+        )
+
+        # If the text format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        generation_kwargs = self.generation_kwargs.copy()
+        text_format = generation_kwargs.pop("text_format", None)
+        if text_format and isinstance(text_format, type) and issubclass(text_format, BaseModel):
+            json_schema = {
+                "format": {
+                    "type": "json_schema",
+                    "name": text_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(text_format),
+                }
+            }
+            # json schema needs to be passed to text parameter instead of text_format
+            generation_kwargs["text"] = json_schema
+
+        # OpenAI/MCP tools are passed as list of dictionaries
+        serialized_tools: dict[str, Any] | list[dict[str, Any]] | None
+        if self.tools and isinstance(self.tools, list) and isinstance(self.tools[0], dict):
+            # mypy can't infer that self.tools is list[dict] here
+            serialized_tools = self.tools
+        else:
+            serialized_tools = serialize_tools_or_toolset(self.tools)  # type: ignore[arg-type]
+
+        return default_to_dict(
+            self,
+            azure_endpoint=self._azure_endpoint,
+            api_key=serialized_api_key,
+            azure_deployment=self._azure_deployment,
+            streaming_callback=callback_name,
+            organization=self.organization,
+            generation_kwargs=generation_kwargs,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            tools=serialized_tools,
+            tools_strict=self.tools_strict,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIResponsesChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        # If api_key is a str, it's a callable (Secrets are handled automatically by default_from_dict)
+        serialized_api_key = data["init_parameters"].get("api_key")
+        if isinstance(serialized_api_key, str):
+            data["init_parameters"]["api_key"] = deserialize_callable(serialized_api_key)
+
+        # we only deserialize the tools if they are haystack tools
+        # because openai tools are not serialized in the same way
+        tools = data["init_parameters"].get("tools")
+        if tools and (
+            isinstance(tools, dict)
+            and tools.get("type") == "haystack.tools.toolset.Toolset"
+            or isinstance(tools, list)
+            and tools[0].get("type") == "haystack.tools.tool.Tool"
+        ):
+            deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        return default_from_dict(cls, data)

haystack/components/generators/chat/fallback.py

@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.components.generators.utils import _normalize_messages
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ChatMessage, StreamingCallbackT
+from haystack.tools import ToolsType
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.deserialization import deserialize_component_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class FallbackChatGenerator:
+    """
+    A chat generator wrapper that tries multiple chat generators sequentially.
+
+    It forwards all parameters transparently to the underlying chat generators and returns the first successful result.
+    Calls chat generators sequentially until one succeeds. Falls back on any exception raised by a generator.
+    If all chat generators fail, it raises a RuntimeError with details.
+
+    Timeout enforcement is fully delegated to the underlying chat generators. The fallback mechanism will only
+    work correctly if the underlying chat generators implement proper timeout handling and raise exceptions
+    when timeouts occur. For predictable latency guarantees, ensure your chat generators:
+    - Support a `timeout` parameter in their initialization
+    - Implement timeout as total wall-clock time (shared deadline for both streaming and non-streaming)
+    - Raise timeout exceptions (e.g., TimeoutError, asyncio.TimeoutError, httpx.TimeoutException) when exceeded
+
+    Note: Most well-implemented chat generators (OpenAI, Anthropic, Cohere, etc.) support timeout parameters
+    with consistent semantics. For HTTP-based LLM providers, a single timeout value (e.g., `timeout=30`)
+    typically applies to all connection phases: connection setup, read, write, and pool. For streaming
+    responses, read timeout is the maximum gap between chunks. For non-streaming, it's the time limit for
+    receiving the complete response.
+
+    Fail over is automatically triggered when a generator raises any exception, including:
+    - Timeout errors (if the generator implements and raises them)
+    - Rate limit errors (429)
+    - Authentication errors (401)
+    - Context length errors (400)
+    - Server errors (500+)
+    - Any other exception
+    """
+
+    def __init__(self, chat_generators: list[ChatGenerator]) -> None:
+        """
+        Creates an instance of FallbackChatGenerator.
+
+        :param chat_generators: A non-empty list of chat generator components to try in order.
+        """
+        if not chat_generators:
+            msg = "'chat_generators' must be a non-empty list"
+            raise ValueError(msg)
+
+        self.chat_generators = list(chat_generators)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component, including nested chat generators."""
+        return default_to_dict(
+            self,
+            chat_generators=[
+                component_to_dict(gen, name=f"chat_generator_{idx}") for idx, gen in enumerate(self.chat_generators)
+            ],
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FallbackChatGenerator:
+        """Rebuild the component from a serialized representation, restoring nested chat generators."""
+        # Reconstruct nested chat generators from their serialized dicts
+        init_params = data.get("init_parameters", {})
+        serialized = init_params.get("chat_generators") or []
+        deserialized: list[Any] = []
+        for g in serialized:
+            # Use the generic component deserializer available in Haystack
+            holder = {"component": g}
+            deserialize_component_inplace(holder, key="component")
+            deserialized.append(holder["component"])
+        init_params["chat_generators"] = deserialized
+        data["init_parameters"] = init_params
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """Warm up all underlying chat generators."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "warm_up"):
+                gen.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """Warm up all underlying chat generators on the serving event loop."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "warm_up_async"):
+                await gen.warm_up_async()
+            elif hasattr(gen, "warm_up"):
+                gen.warm_up()
+
+    def close(self) -> None:
+        """Release the underlying chat generators' resources."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "close"):
+                gen.close()
+
+    async def close_async(self) -> None:
+        """Release the underlying chat generators' async resources."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "close_async"):
+                await gen.close_async()
+            elif hasattr(gen, "close"):
+                gen.close()
+
+    def _run_single_sync(
+        self,
+        gen: Any,
+        messages: list[ChatMessage],
+        generation_kwargs: dict[str, Any] | None,
+        tools: ToolsType | None,
+        streaming_callback: StreamingCallbackT | None,
+    ) -> dict[str, Any]:
+        return gen.run(
+            messages=messages, generation_kwargs=generation_kwargs, tools=tools, streaming_callback=streaming_callback
+        )
+
+    async def _run_single_async(
+        self,
+        gen: Any,
+        messages: list[ChatMessage],
+        generation_kwargs: dict[str, Any] | None,
+        tools: ToolsType | None,
+        streaming_callback: StreamingCallbackT | None,
+    ) -> dict[str, Any]:
+        return await _execute_component_async(
+            gen,
+            messages=messages,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            streaming_callback=streaming_callback,
+        )
+
+    @component.output_types(replies=list[ChatMessage], meta=dict[str, Any])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> dict[str, list[ChatMessage] | dict[str, Any]]:
+        """
+        Execute chat generators sequentially until one succeeds.
+
+        :param messages: The conversation history as a list of ChatMessage instances.
+        :param generation_kwargs: Optional parameters for the chat generator (e.g., temperature, max_tokens).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for function calling capabilities.
+        :param streaming_callback: Optional callable for handling streaming responses.
+        :returns: A dictionary with:
+            - "replies": Generated ChatMessage instances from the first successful generator.
+            - "meta": Execution metadata including successful_chat_generator_index, successful_chat_generator_class,
+              total_attempts, failed_chat_generators, plus any metadata from the successful generator.
+        :raises RuntimeError: If all chat generators fail.
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+
+        failed: list[str] = []
+        last_error: BaseException | None = None
+
+        for idx, gen in enumerate(self.chat_generators):
+            gen_name = gen.__class__.__name__
+            try:
+                result = self._run_single_sync(gen, messages, generation_kwargs, tools, streaming_callback)
+                replies = result.get("replies", [])
+                meta = dict(result.get("meta", {}))
+                meta.update(
+                    {
+                        "successful_chat_generator_index": idx,
+                        "successful_chat_generator_class": gen_name,
+                        "total_attempts": idx + 1,
+                        "failed_chat_generators": failed,
+                    }
+                )
+                return {"replies": replies, "meta": meta}
+            except Exception as e:  # noqa: BLE001 - fallback logic should handle any exception
+                logger.warning(
+                    "ChatGenerator {chat_generator} failed with error: {error}", chat_generator=gen_name, error=e
+                )
+                failed.append(gen_name)
+                last_error = e
+
+        failed_names = ", ".join(failed)
+        msg = (
+            f"All {len(self.chat_generators)} chat generators failed. "
+            f"Last error: {last_error}. Failed chat generators: [{failed_names}]"
+        )
+        raise RuntimeError(msg)
+
+    @component.output_types(replies=list[ChatMessage], meta=dict[str, Any])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> dict[str, list[ChatMessage] | dict[str, Any]]:
+        """
+        Asynchronously execute chat generators sequentially until one succeeds.
+
+        :param messages: The conversation history as a list of ChatMessage instances.
+        :param generation_kwargs: Optional parameters for the chat generator (e.g., temperature, max_tokens).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for function calling capabilities.
+        :param streaming_callback: Optional callable for handling streaming responses.
+        :returns: A dictionary with:
+            - "replies": Generated ChatMessage instances from the first successful generator.
+            - "meta": Execution metadata including successful_chat_generator_index, successful_chat_generator_class,
+              total_attempts, failed_chat_generators, plus any metadata from the successful generator.
+        :raises RuntimeError: If all chat generators fail.
+        """
+        await self.warm_up_async()
+
+        messages = _normalize_messages(messages)
+
+        failed: list[str] = []
+        last_error: BaseException | None = None
+
+        for idx, gen in enumerate(self.chat_generators):
+            gen_name = gen.__class__.__name__
+            try:
+                result = await self._run_single_async(gen, messages, generation_kwargs, tools, streaming_callback)
+                replies = result.get("replies", [])
+                meta = dict(result.get("meta", {}))
+                meta.update(
+                    {
+                        "successful_chat_generator_index": idx,
+                        "successful_chat_generator_class": gen_name,
+                        "total_attempts": idx + 1,
+                        "failed_chat_generators": failed,
+                    }
+                )
+                return {"replies": replies, "meta": meta}
+            except Exception as e:  # noqa: BLE001 - fallback logic should handle any exception
+                logger.warning(
+                    "ChatGenerator {chat_generator} failed with error: {error}", chat_generator=gen_name, error=e
+                )
+                failed.append(gen_name)
+                last_error = e
+
+        failed_names = ", ".join(failed)
+        msg = (
+            f"All {len(self.chat_generators)} chat generators failed. "
+            f"Last error: {last_error}. Failed chat generators: [{failed_names}]"
+        )
+        raise RuntimeError(msg)

haystack/components/generators/chat/llm.py

@@ -0,0 +1,205 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Literal
+
+from haystack import component, logging
+from haystack.components.agents.agent import Agent
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage, StreamingCallbackT
+from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
+from haystack.utils.deserialization import deserialize_component_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLM(Agent):
+    """
+    A text generation component powered by a large language model.
+
+    The LLM component is a simplified version of the Agent that focuses solely on text generation
+    without tool usage. It processes messages and returns a single response from the language model.
+
+    ### Usage examples
+    ```python
+    from haystack.components.generators.chat import LLM
+    from haystack.components.generators.chat import OpenAIChatGenerator
+
+    llm = LLM(
+        chat_generator=OpenAIChatGenerator(),
+        system_prompt="You are a helpful translation assistant.",
+        user_prompt="Summarize the following document: {{ document }}",
+        required_variables=["document"],
+    )
+
+    result = llm.run(document="The weather is lovely today and the sun is shining. ")
+    print(result["last_message"].text)
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator,
+        system_prompt: str | None = None,
+        user_prompt: str | None = None,
+        required_variables: list[str] | Literal["*"] = "*",
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> None:
+        """
+        Initialize the LLM component.
+
+        :param chat_generator: An instance of the chat generator that the LLM should use.
+        :param system_prompt: System prompt for the LLM. Can be a plain string template or a Jinja2 message template.
+        :param user_prompt: User prompt for the LLM. This prompt is appended to the messages provided at
+            runtime. Can be a plain string template or a Jinja2 message template. If it contains template variables
+            (e.g., `{{ variable_name }}`), they become inputs to the component. If omitted or if there are no
+            template variables, `messages` must be provided at runtime instead.
+        :param required_variables:
+            Variables that must be provided as input to `user_prompt` or `system_prompt`.
+            If a variable listed as required is not provided, an exception is raised.
+            If set to `"*"`, all variables found in the prompt are required. Defaults to `"*"`.
+            Only relevant when `user_prompt` or `system_prompt` contains template variables.
+        :param streaming_callback: A callback that will be invoked when a response is streamed from the LLM.
+        :raises ValueError: If user_prompt contains template variables but required_variables is an empty list.
+        """
+        super(LLM, self).__init__(  # noqa: UP008
+            chat_generator=chat_generator,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            required_variables=required_variables,
+            streaming_callback=streaming_callback,
+        )
+        if self._user_chat_prompt_builder is None or len(self._user_chat_prompt_builder.variables) == 0:
+            # This means user_prompt is empty or has no template variables.
+            # To ensure properly scheduling we then require messages to be passed at runtime.
+            component.set_input_type(self, "messages", list[ChatMessage])
+        else:
+            # user prompt was provided with variables
+            if isinstance(required_variables, list) and len(required_variables) == 0:
+                raise ValueError(
+                    "required_variables must not be empty. Set it to '*' to require all variables, "
+                    "or provide a non-empty list of variable names."
+                )
+            component.set_input_type(self, "messages", list[ChatMessage], None)
+
+        # The Agent base class declares `step_count` and `tool_call_counts` as outputs, but an LLM never has tools
+        # and always runs exactly one step — those values are uninformative, so drop them from the public surface.
+        # `token_usage` is still meaningful and stays exposed.
+        component.set_output_types(
+            self, messages=list[ChatMessage], last_message=ChatMessage, token_usage=dict[str, Any]
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize the LLM component to a dictionary.
+
+        :return: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self.chat_generator, name="chat_generator"),
+            system_prompt=self.system_prompt,
+            user_prompt=self.user_prompt,
+            required_variables=self.required_variables,
+            streaming_callback=serialize_callable(self.streaming_callback) if self.streaming_callback else None,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLM":
+        """
+        Deserialize the LLM from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :return: Deserialized LLM instance.
+        """
+        init_params = data.get("init_parameters", {})
+
+        deserialize_component_inplace(init_params, key="chat_generator")
+
+        if init_params.get("streaming_callback") is not None:
+            init_params["streaming_callback"] = deserialize_callable(init_params["streaming_callback"])
+
+        return default_from_dict(cls, data)
+
+    def run(  # type: ignore[override]  # `messages` is in **kwargs to allow dynamic required/optional status
+        self,
+        *,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Process messages and generate a response from the language model.
+
+        :param messages: Optional list of ChatMessage objects to prepend to the conversation. Whether this is
+            required or optional depends on the `user_prompt` configuration: if `user_prompt` has no template
+            variables, `messages` must be provided. Passed via `**kwargs`.
+        :param streaming_callback: A callback that will be invoked when a response is streamed from the LLM.
+        :param generation_kwargs: Additional keyword arguments for the underlying chat generator. These parameters
+            will override the parameters passed during component initialization.
+        :param kwargs: Additional keyword arguments. These are used to fill template variables in `user_prompt` or
+            `system_prompt` (the keys must match template variable names).
+        :returns:
+            A dictionary with the following keys:
+            - "messages": List of all messages exchanged during the LLM's run.
+            - "last_message": The last message exchanged during the LLM's run.
+            - "token_usage": Token usage from the LLM call (e.g. prompt_tokens, completion_tokens). Empty if the
+              chat generator did not return usage data.
+        """
+        # `messages` is intentionally omitted from the signature so the framework can treat it as required
+        # or optional depending on init configuration. See __init__ for details.
+        messages = kwargs.pop("messages", None)
+        result = super(LLM, self).run(  # noqa: UP008
+            messages=messages or [],
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            **kwargs,
+        )
+        # Inherited Agent-internal bookkeeping that isn't useful at the LLM surface.
+        result.pop("step_count", None)
+        result.pop("tool_call_counts", None)
+        return result
+
+    async def run_async(  # type: ignore[override]  # `messages` is in **kwargs to allow dynamic required/optional status
+        self,
+        *,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Asynchronously process messages and generate a response from the language model.
+
+        :param messages: Optional list of ChatMessage objects to prepend to the conversation. Whether this is
+            required or optional depends on the `user_prompt` configuration: if `user_prompt` has no template
+            variables, `messages` must be provided. Passed via `**kwargs`.
+        :param streaming_callback: An asynchronous callback that will be invoked when a response is streamed
+            from the LLM.
+        :param generation_kwargs: Additional keyword arguments for the underlying chat generator. These parameters
+            will override the parameters passed during component initialization.
+        :param kwargs: Additional keyword arguments. These are used to fill template variables in `user_prompt` or
+            `system_prompt` (the keys must match template variable names).
+        :returns:
+            A dictionary with the following keys:
+            - "messages": List of all messages exchanged during the LLM's run.
+            - "last_message": The last message exchanged during the LLM's run.
+            - "token_usage": Token usage from the LLM call (e.g. prompt_tokens, completion_tokens). Empty if the
+              chat generator did not return usage data.
+        """
+        # `messages` is intentionally omitted from the signature so the framework can treat it as required
+        # or optional depending on init configuration. See __init__ for details.
+        messages = kwargs.pop("messages", None)
+        result = await super(LLM, self).run_async(  # noqa: UP008
+            messages=messages or [],
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            **kwargs,
+        )
+        # Inherited Agent-internal bookkeeping that isn't useful at the LLM surface.
+        result.pop("step_count", None)
+        result.pop("tool_call_counts", None)
+        return result

haystack/components/generators/chat/mock.py

@@ -0,0 +1,374 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable, Sequence
+from dataclasses import replace
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.utils import _normalize_messages
+from haystack.dataclasses import (
+    ChatMessage,
+    ChatRole,
+    ComponentInfo,
+    FinishReason,
+    StreamingCallbackT,
+    StreamingChunk,
+    select_streaming_callback,
+)
+from haystack.dataclasses.streaming_chunk import ToolCallDelta, _invoke_streaming_callback
+from haystack.tools import ToolsType
+from haystack.utils import deserialize_callable, serialize_callable
+
+logger = logging.getLogger(__name__)
+
+# A callable that derives a response from the input messages. It receives the (normalized) list of input
+# `ChatMessage` objects and returns either the text of the assistant reply or a full `ChatMessage`.
+ResponseFn = Callable[[list[ChatMessage]], str | ChatMessage]
+
+
+@component
+class MockChatGenerator:
+    """
+    A Chat Generator that returns predefined responses without calling any API.
+
+    It is a drop-in replacement for real Chat Generators (such as `OpenAIChatGenerator`) in tests, smoke tests, and
+    quick prototypes. It implements the same interface (`run`, `run_async`, streaming, serialization) but never
+    contacts an external service, so it is fully deterministic and free to run.
+
+    The response is selected based on how the component is configured:
+
+    - **Fixed response**: pass a single string or `ChatMessage`. The same reply is returned on every call.
+      Any `ChatMessage` passed as a response must have the `assistant` role.
+    - **Cycling responses**: pass a list of strings and/or `ChatMessage` objects. Each call returns the next item,
+      wrapping around to the start once the list is exhausted. This is useful to drive multi-step flows such as
+      Agents, where the first call returns a tool call and a later call returns the final answer.
+    - **Dynamic response**: pass a `response_fn` callable that receives the input messages and returns the reply.
+      This is useful when the reply should depend on the input, for example to echo back part of the prompt.
+    - **Echo (default)**: with no configuration, the component echoes back the text of the last message that has
+      text content. This makes it usable out of the box for quick prototyping.
+
+    Pass `ChatMessage` objects (rather than plain strings) to return tool calls or reasoning content, which is handy
+    for exercising tool-calling pipelines without a real model.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.generators.chat import MockChatGenerator
+    from haystack.dataclasses import ChatMessage, ToolCall
+
+    # Fixed response
+    generator = MockChatGenerator(responses="Hello, this is a mock response.")
+    result = generator.run([ChatMessage.from_user("Hi!")])
+    print(result["replies"][0].text)  # "Hello, this is a mock response."
+
+    # Cycling responses to drive an Agent-like loop
+    generator = MockChatGenerator(
+        responses=[
+            ChatMessage.from_assistant(tool_calls=[ToolCall(tool_name="search", arguments={"query": "Haystack"})]),
+            "Here is the final answer.",
+        ]
+    )
+    ```
+    """
+
+    def __init__(
+        self,
+        responses: str | ChatMessage | Sequence[str | ChatMessage] | None = None,
+        *,
+        response_fn: ResponseFn | None = None,
+        model: str = "mock-model",
+        meta: dict[str, Any] | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> None:
+        """
+        Creates an instance of MockChatGenerator.
+
+        :param responses: The predefined response(s) to return. Accepts a single string or `ChatMessage` (returned on
+            every call), or a non-empty list of strings and/or `ChatMessage` objects that are returned in order,
+            cycling back to the start once exhausted. Strings are wrapped into assistant `ChatMessage` objects, and any
+            `ChatMessage` passed must have the `assistant` role. Mutually exclusive with `response_fn`. If neither is
+            provided, the component echoes the last message with text content.
+        :param response_fn: An optional callable that receives the input messages and returns the reply as a string or
+            an assistant `ChatMessage`. Use this for input-dependent responses. Mutually exclusive with `responses`. To
+            support serialization, pass a named function (lambdas and nested functions cannot be serialized).
+        :param model: The model name reported in the response metadata. Purely cosmetic; no model is loaded.
+        :param meta: Additional metadata merged into the `meta` of every returned `ChatMessage`. A per-response
+            `ChatMessage`'s own metadata takes precedence over this value.
+        :param streaming_callback: An optional callback invoked with `StreamingChunk` objects reconstructed from the
+            predefined response. It lets the mock exercise streaming code paths without a real model.
+        :raises ValueError: If both `responses` and `response_fn` are provided, if `responses` is an empty list, or if
+            a `ChatMessage` response does not have the `assistant` role.
+        """
+        if responses is not None and response_fn is not None:
+            raise ValueError("Pass either 'responses' or 'response_fn', not both.")
+
+        self._responses = self._normalize_responses(responses)
+        self.response_fn = response_fn
+        self.model = model
+        self.meta = meta or {}
+        self.streaming_callback = streaming_callback
+        self._call_count = 0
+        self._is_warmed_up = False
+
+    @staticmethod
+    def _normalize_responses(
+        responses: str | ChatMessage | Sequence[str | ChatMessage] | None,
+    ) -> list[ChatMessage] | None:
+        """Normalize the `responses` argument into a non-empty list of `ChatMessage`, or `None` for echo mode."""
+        if responses is None:
+            return None
+
+        items: list[str | ChatMessage]
+        if isinstance(responses, (str, ChatMessage)):
+            items = [responses]
+        elif isinstance(responses, Sequence):
+            items = list(responses)
+        else:
+            raise TypeError(f"'responses' must be a string, ChatMessage, or a sequence of them, got {type(responses)}.")
+
+        if len(items) == 0:
+            raise ValueError("'responses' must not be an empty list.")
+
+        normalized: list[ChatMessage] = []
+        for item in items:
+            if isinstance(item, str):
+                normalized.append(ChatMessage.from_assistant(item))
+            elif isinstance(item, ChatMessage):
+                if item.role != ChatRole.ASSISTANT:
+                    raise ValueError(
+                        f"Each ChatMessage response must have the 'assistant' role, got '{item.role.value}'."
+                    )
+                normalized.append(item)
+            else:
+                raise TypeError(f"Each response must be a string or ChatMessage, got {type(item)}.")
+        return normalized
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        responses = [msg.to_dict() for msg in self._responses] if self._responses is not None else None
+        response_fn = serialize_callable(self.response_fn) if self.response_fn is not None else None
+        streaming_callback = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        return default_to_dict(
+            self,
+            responses=responses,
+            response_fn=response_fn,
+            model=self.model,
+            meta=self.meta,
+            streaming_callback=streaming_callback,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> MockChatGenerator:
+        """Deserialize the component from a dictionary."""
+        init_params = data.get("init_parameters", {})
+        responses = init_params.get("responses")
+        if responses is not None:
+            init_params["responses"] = [ChatMessage.from_dict(msg) for msg in responses]
+        response_fn = init_params.get("response_fn")
+        if response_fn:
+            init_params["response_fn"] = deserialize_callable(response_fn)
+        streaming_callback = init_params.get("streaming_callback")
+        if streaming_callback:
+            init_params["streaming_callback"] = deserialize_callable(streaming_callback)
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """No-op warm up, provided for interface compatibility with real Chat Generators."""
+        self._is_warmed_up = True
+
+    @staticmethod
+    def _echo_text(messages: list[ChatMessage]) -> str | None:
+        """Return the text of the last message that has text content, for echo mode."""
+        for message in reversed(messages):
+            if message.text:
+                return message.text
+        return None
+
+    @staticmethod
+    def _coerce_to_message(result: str | ChatMessage) -> ChatMessage:
+        """Turn the output of `response_fn` into a `ChatMessage`, wrapping strings and requiring the assistant role."""
+        if isinstance(result, str):
+            return ChatMessage.from_assistant(result)
+        if isinstance(result, ChatMessage):
+            if result.role != ChatRole.ASSISTANT:
+                raise ValueError(f"'response_fn' must return an assistant ChatMessage, got '{result.role.value}'.")
+            return result
+        raise TypeError(f"'response_fn' must return a string or ChatMessage, got {type(result)}.")
+
+    @staticmethod
+    def _estimate_usage(messages: list[ChatMessage], reply: ChatMessage) -> dict[str, int]:
+        """
+        Roughly estimate token usage as whitespace-separated word counts.
+
+        This is an approximation (not real tokenization) intended to give downstream code realistic-looking metadata.
+        """
+        prompt_tokens = sum(len((message.text or "").split()) for message in messages)
+        completion_tokens = len((reply.text or "").split())
+        return {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        }
+
+    def _build_meta(self, messages: list[ChatMessage], base: ChatMessage) -> dict[str, Any]:
+        """Build the metadata attached to the returned reply, merging defaults, init meta, and per-response meta."""
+        meta: dict[str, Any] = {
+            "model": self.model,
+            "index": 0,
+            "finish_reason": "tool_calls" if base.tool_calls else "stop",
+            "usage": self._estimate_usage(messages, base),
+        }
+        meta.update(self.meta)
+        meta.update(base.meta)
+        return meta
+
+    def _build_reply(self, messages: list[ChatMessage]) -> ChatMessage | None:
+        """Select and finalize the reply for the given input messages. Returns `None` when there is nothing to echo."""
+        if self.response_fn is not None:
+            base = self._coerce_to_message(self.response_fn(messages))
+        elif self._responses is not None:
+            base = self._responses[self._call_count % len(self._responses)]
+            self._call_count += 1
+        else:
+            text = self._echo_text(messages)
+            if text is None:
+                return None
+            base = ChatMessage.from_assistant(text)
+
+        return replace(base, _meta=self._build_meta(messages, base))
+
+    def _make_chunks(self, reply: ChatMessage) -> list[StreamingChunk]:
+        """Reconstruct streaming chunks from a finalized reply so streaming callbacks can be exercised."""
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+
+        # Stream the text content word by word in content block 0.
+        parts = re.findall(r"\S+\s*", reply.text) if reply.text else []
+        for idx, part in enumerate(parts):
+            chunks.append(
+                StreamingChunk(
+                    content=part, component_info=component_info, index=0, start=(idx == 0), meta={"model": self.model}
+                )
+            )
+
+        # Stream each tool call in its own content block.
+        block_index = 1 if parts else 0
+        for tool_call in reply.tool_calls:
+            chunks.append(
+                StreamingChunk(
+                    content="",
+                    component_info=component_info,
+                    index=block_index,
+                    start=True,
+                    tool_calls=[
+                        ToolCallDelta(
+                            index=block_index,
+                            tool_name=tool_call.tool_name,
+                            arguments=json.dumps(tool_call.arguments),
+                            id=tool_call.id,
+                        )
+                    ],
+                    meta={"model": self.model},
+                )
+            )
+            block_index += 1
+
+        if not chunks:
+            chunks.append(
+                StreamingChunk(content="", component_info=component_info, index=0, meta={"model": self.model})
+            )
+
+        finish_reason: FinishReason = "tool_calls" if reply.tool_calls else "stop"
+        last = chunks[-1]
+        chunks[-1] = replace(last, finish_reason=finish_reason, meta={**last.meta, "finish_reason": finish_reason})
+        return chunks
+
+    @component.output_types(replies=list[ChatMessage])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,  # noqa: ARG002
+        *,
+        tools: ToolsType | None = None,  # noqa: ARG002
+        tools_strict: bool | None = None,  # noqa: ARG002
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Return a predefined reply for the given messages without calling any API.
+
+        The signature mirrors `OpenAIChatGenerator.run` so the mock can be used as a positional drop-in replacement.
+
+        :param messages: The conversation history as a list of `ChatMessage` instances or a single string.
+        :param streaming_callback: An optional callback invoked with reconstructed `StreamingChunk` objects. Overrides
+            the callback set at initialization.
+        :param generation_kwargs: Accepted for interface compatibility and ignored.
+        :param tools: Accepted for interface compatibility and ignored.
+        :param tools_strict: Accepted for interface compatibility and ignored.
+        :returns: A dictionary with a single key `replies` containing the predefined reply as a list of one
+            `ChatMessage` (empty in echo mode when there is no message to echo).
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=False
+        )
+
+        reply = self._build_reply(messages)
+        if reply is None:
+            return {"replies": []}
+
+        if streaming_callback is not None:
+            for chunk in self._make_chunks(reply):
+                streaming_callback(chunk)
+
+        return {"replies": [reply]}
+
+    @component.output_types(replies=list[ChatMessage])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,  # noqa: ARG002
+        *,
+        tools: ToolsType | None = None,  # noqa: ARG002
+        tools_strict: bool | None = None,  # noqa: ARG002
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Asynchronously return a predefined reply for the given messages without calling any API.
+
+        The signature mirrors `OpenAIChatGenerator.run_async` so the mock can be used as a positional drop-in
+        replacement.
+
+        :param messages: The conversation history as a list of `ChatMessage` instances or a single string.
+        :param streaming_callback: An optional callback invoked with reconstructed `StreamingChunk` objects. Overrides
+            the callback set at initialization.
+        :param generation_kwargs: Accepted for interface compatibility and ignored.
+        :param tools: Accepted for interface compatibility and ignored.
+        :param tools_strict: Accepted for interface compatibility and ignored.
+        :returns: A dictionary with a single key `replies` containing the predefined reply as a list of one
+            `ChatMessage` (empty in echo mode when there is no message to echo).
+        """
+        if not self._is_warmed_up:
+            self.warm_up()
+
+        messages = _normalize_messages(messages)
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=True
+        )
+
+        reply = self._build_reply(messages)
+        if reply is None:
+            return {"replies": []}
+
+        if streaming_callback is not None:
+            for chunk in self._make_chunks(reply):
+                await _invoke_streaming_callback(streaming_callback, chunk)
+
+        return {"replies": [reply]}

haystack/components/generators/chat/openai.py

@@ -0,0 +1,795 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+import os
+from datetime import datetime
+from typing import Any, ClassVar
+
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from openai.lib._pydantic import to_strict_json_schema
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    ChatCompletionMessageCustomToolCall,
+    ParsedChatCompletion,
+    ParsedChatCompletionMessage,
+)
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.utils import (
+    _convert_streaming_chunks_to_chat_message,
+    _normalize_messages,
+    _serialize_object,
+)
+from haystack.dataclasses import (
+    ChatMessage,
+    ComponentInfo,
+    FinishReason,
+    StreamingCallbackT,
+    StreamingChunk,
+    SyncStreamingCallbackT,
+    ToolCall,
+    ToolCallDelta,
+    select_streaming_callback,
+)
+from haystack.dataclasses.streaming_chunk import _invoke_streaming_callback
+from haystack.tools import (
+    ToolsType,
+    _check_duplicate_tool_names,
+    deserialize_tools_or_toolset_inplace,
+    flatten_tools_or_toolsets,
+    serialize_tools_or_toolset,
+    warm_up_tools,
+)
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class OpenAIChatGenerator:
+    """
+    Completes chats using OpenAI's large language models (LLMs).
+
+    It works with the gpt-4 and gpt-5 series models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.ChatCompletion.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+
+    ### Usage example
+    ```python
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = OpenAIChatGenerator()
+    response = client.run(messages)
+    print(response)
+    ```
+    Output:
+    ```
+    {'replies':
+        [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=
+        [TextContent(text="Natural Language Processing (NLP) is a branch of artificial intelligence
+            that focuses on enabling computers to understand, interpret, and generate human language in
+            a way that is meaningful and useful.")],
+         _name=None,
+         _meta={'model': 'gpt-5-mini', 'index': 0, 'finish_reason': 'stop',
+         'usage': {'prompt_tokens': 15, 'completion_tokens': 36, 'total_tokens': 51}})
+        ]
+    }
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5",
+        "gpt-5.1",
+        "gpt-5.2",
+        "gpt-5.2-pro",
+        "gpt-5.4",
+        "gpt-5-pro",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-4.1-nano",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4",
+        "gpt-3.5-turbo",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://developers.openai.com/api/docs/models for the full list and snapshot IDs."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
+        model: str = "gpt-5-mini",
+        streaming_callback: StreamingCallbackT | None = None,
+        api_base_url: str | None = None,
+        organization: str | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Creates an instance of OpenAIChatGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-5-mini
+
+        Before initializing the component, you can set the 'OPENAI_TIMEOUT' and 'OPENAI_MAX_RETRIES'
+        environment variables to override the `timeout` and `max_retries` parameters respectively
+        in the OpenAI client.
+
+        :param api_key: The OpenAI API key.
+            You can set it with an environment variable `OPENAI_API_KEY`, or pass with this parameter
+            during initialization.
+        :param model: The name of the model to use.
+        :param streaming_callback: A callback function that is called when a new token is received from the stream.
+            The callback function accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param api_base_url: An optional base URL.
+        :param organization: Your organization ID, defaults to `None`. See
+        [production best practices](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent directly to
+            the OpenAI endpoint. See OpenAI [documentation](https://platform.openai.com/docs/api-reference/chat) for
+            more details.
+            Some of the supported parameters:
+            - `max_completion_tokens`: An upper bound for the number of tokens that can be generated for a completion,
+                including visible output tokens and reasoning tokens.
+            - `temperature`: What sampling temperature to use. Higher values mean the model will take more risks.
+                Try 0.9 for more creative applications and 0 (argmax sampling) for ones with a well-defined answer.
+            - `top_p`: An alternative to sampling with temperature, called nucleus sampling, where the model
+                considers the results of the tokens with top_p probability mass. For example, 0.1 means only the tokens
+                comprising the top 10% probability mass are considered.
+            - `n`: How many completions to generate for each prompt. For example, if the LLM gets 3 prompts and n is 2,
+                it will generate two completions for each of the three prompts, ending up with 6 completions in total.
+            - `stop`: One or more sequences after which the LLM should stop generating tokens.
+            - `presence_penalty`: What penalty to apply if a token is already present at all. Bigger values mean
+                the model will be less likely to repeat the same token in the text.
+            - `frequency_penalty`: What penalty to apply if a token has already been generated in the text.
+                Bigger values mean the model will be less likely to repeat the same token in the text.
+            - `logit_bias`: Add a logit bias to specific tokens. The keys of the dictionary are tokens, and the
+                values are the bias to add to that token.
+            - `response_format`: A JSON schema or a Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+                Notes:
+                - This parameter accepts Pydantic models and JSON schemas for latest models starting from GPT-4o.
+                  Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                  For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+                - For structured outputs with streaming,
+                  the `response_format` must be a JSON schema and not a Pydantic model.
+        :param timeout:
+            Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries:
+            Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+
+        """
+        self.api_key = api_key
+        self.model = model
+        self.generation_kwargs = generation_kwargs or {}
+        self.streaming_callback = streaming_callback
+        self.api_base_url = api_base_url
+        self.organization = organization
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.tools = tools  # Store tools as-is, whether it's a list or a Toolset
+        self.tools_strict = tools_strict
+        self.http_client_kwargs = http_client_kwargs
+        # Check for duplicate tool names
+        _check_duplicate_tool_names(flatten_tools_or_toolsets(self.tools))
+
+        self.client: OpenAI | None = None
+        self.async_client: AsyncOpenAI | None = None
+        self._tools_warmed_up = False
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_key": self.api_key.resolve_value(),
+            "organization": self.organization,
+            "base_url": self.api_base_url,
+            "timeout": timeout,
+            "max_retries": max_retries,
+        }
+
+    def _warm_up_tools(self) -> None:
+        if not self._tools_warmed_up:
+            warm_up_tools(self.tools)
+            self._tools_warmed_up = True
+
+    def warm_up(self) -> None:
+        """
+        Warm up the tools and initialize the synchronous OpenAI client.
+        """
+        self._warm_up_tools()
+        if self.client is None:
+            self.client = OpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Warm up the tools and initialize the asynchronous OpenAI client on the serving event loop.
+        """
+        self._warm_up_tools()
+        if self.async_client is None:
+            self.async_client = AsyncOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        generation_kwargs = self.generation_kwargs.copy()
+        response_format = generation_kwargs.get("response_format")
+
+        # If the response format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        if response_format and isinstance(response_format, type) and issubclass(response_format, BaseModel):
+            json_schema = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": response_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(response_format),
+                },
+            }
+            generation_kwargs["response_format"] = json_schema
+
+        return default_to_dict(
+            self,
+            model=self.model,
+            streaming_callback=callback_name,
+            api_base_url=self.api_base_url,
+            organization=self.organization,
+            generation_kwargs=generation_kwargs,
+            api_key=self.api_key,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            tools=serialize_tools_or_toolset(self.tools),
+            tools_strict=self.tools_strict,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OpenAIChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        return default_from_dict(cls, data)
+
+    @component.output_types(replies=list[ChatMessage])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        *,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Invokes chat completion based on the provided messages and generation parameters.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages. If a string is provided, it is converted
+            to a list containing a ChatMessage with user role.
+        :param streaming_callback:
+            A callback function that is called when a new token is received from the stream.
+        :param generation_kwargs:
+            Additional keyword arguments for text generation. These parameters will
+            override the parameters passed during component initialization.
+            For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+            If set, it will override the `tools_strict` parameter set during component initialization.
+
+        :returns:
+            A dictionary with the following key:
+            - `replies`: A list containing the generated responses as ChatMessage instances.
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+
+        if len(messages) == 0:
+            return {"replies": []}
+
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=False
+        )
+        chat_completion: Stream[ChatCompletionChunk] | ChatCompletion | ParsedChatCompletion
+
+        api_args = self._prepare_api_call(
+            messages=messages,
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            tools_strict=tools_strict,
+        )
+        openai_endpoint = api_args.pop("openai_endpoint")
+        assert self.client is not None  # mypy: client is built by warm_up above
+        openai_endpoint_method = getattr(self.client.chat.completions, openai_endpoint)
+        chat_completion = openai_endpoint_method(**api_args)
+
+        if streaming_callback is not None:
+            completions = self._handle_stream_response(
+                # we cannot check isinstance(chat_completion, Stream) because some observability tools wrap Stream
+                # and return a different type. See https://github.com/deepset-ai/haystack/issues/9014.
+                chat_completion,  # type: ignore
+                streaming_callback,
+            )
+
+        else:
+            assert isinstance(chat_completion, ChatCompletion), "Unexpected response type for non-streaming request."
+            completions = [
+                _convert_chat_completion_to_chat_message(chat_completion, choice) for choice in chat_completion.choices
+            ]
+
+        # before returning, do post-processing of the completions
+        for message in completions:
+            _check_finish_reason(message.meta)
+
+        return {"replies": completions}
+
+    @component.output_types(replies=list[ChatMessage])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        *,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Asynchronously invokes chat completion based on the provided messages and generation parameters.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in async code.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages. If a string is provided, it is converted
+            to a list containing a ChatMessage with user role.
+        :param streaming_callback:
+            A callback function that is called when a new token is received from the stream. Async callbacks are
+            preferred; a sync callback is accepted but will run synchronously on the event loop and may block it.
+        :param generation_kwargs:
+            Additional keyword arguments for text generation. These parameters will
+            override the parameters passed during component initialization.
+            For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+            If set, it will override the `tools_strict` parameter set during component initialization.
+
+        :returns:
+            A dictionary with the following key:
+            - `replies`: A list containing the generated responses as ChatMessage instances.
+        """
+        await self.warm_up_async()
+
+        messages = _normalize_messages(messages)
+
+        # validate and select the streaming callback
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=True
+        )
+        chat_completion: AsyncStream[ChatCompletionChunk] | ChatCompletion | ParsedChatCompletion
+
+        if len(messages) == 0:
+            return {"replies": []}
+
+        api_args = self._prepare_api_call(
+            messages=messages,
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            tools_strict=tools_strict,
+        )
+
+        openai_endpoint = api_args.pop("openai_endpoint")
+        assert self.async_client is not None  # mypy: async_client is built by warm_up_async above
+        openai_endpoint_method = getattr(self.async_client.chat.completions, openai_endpoint)
+        chat_completion = await openai_endpoint_method(**api_args)
+
+        if streaming_callback is not None:
+            completions = await self._handle_async_stream_response(
+                # we cannot check isinstance(chat_completion, AsyncStream) because some observability tools wrap
+                # AsyncStream and return a different type. See https://github.com/deepset-ai/haystack/issues/9014.
+                chat_completion,  # type: ignore
+                streaming_callback,
+            )
+
+        else:
+            assert isinstance(chat_completion, ChatCompletion), "Unexpected response type for non-streaming request."
+            completions = [
+                _convert_chat_completion_to_chat_message(chat_completion, choice) for choice in chat_completion.choices
+            ]
+
+        # before returning, do post-processing of the completions
+        for message in completions:
+            _check_finish_reason(message.meta)
+
+        return {"replies": completions}
+
+    def _prepare_api_call(  # noqa: PLR0913
+        self,
+        *,
+        messages: list[ChatMessage],
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, Any]:
+        # update generation kwargs by merging with the generation kwargs passed to the run method
+        generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
+
+        is_streaming = streaming_callback is not None
+        num_responses = generation_kwargs.pop("n", 1)
+
+        if is_streaming and num_responses > 1:
+            raise ValueError("Cannot stream multiple responses, please set n=1.")
+        response_format = generation_kwargs.pop("response_format", None)
+
+        # adapt ChatMessage(s) to the format expected by the OpenAI API
+        openai_formatted_messages = [message.to_openai_dict_format() for message in messages]
+
+        flattened_tools = flatten_tools_or_toolsets(tools or self.tools)
+        tools_strict = tools_strict if tools_strict is not None else self.tools_strict
+        _check_duplicate_tool_names(flattened_tools)
+
+        openai_tools = {}
+        if flattened_tools:
+            tool_definitions = []
+            for t in flattened_tools:
+                function_spec = {**t.tool_spec}
+                if tools_strict:
+                    function_spec["strict"] = True
+                    function_spec["parameters"] = _make_schema_strict(function_spec["parameters"])
+                tool_definitions.append({"type": "function", "function": function_spec})
+            openai_tools = {"tools": tool_definitions}
+
+        base_args = {
+            "model": self.model,
+            "messages": openai_formatted_messages,
+            "n": num_responses,
+            **openai_tools,
+            **generation_kwargs,
+        }
+
+        if response_format and not is_streaming:
+            # for structured outputs without streaming, we use openai's parse endpoint
+            # Note: `stream` cannot be passed to chat.completions.parse
+            # we pass a key `openai_endpoint` as a hint to the run method to use the parse endpoint
+            # this key will be removed before the API call is made
+            return {**base_args, "response_format": response_format, "openai_endpoint": "parse"}
+
+        # for structured outputs with streaming, we use openai's create endpoint
+        # we pass a key `openai_endpoint` as a hint to the run method to use the create endpoint
+        # this key will be removed before the API call is made
+        final_args = {**base_args, "stream": is_streaming, "openai_endpoint": "create"}
+
+        # We only set the response_format parameter if it's not None since None is not a valid value in the API.
+        if response_format:
+            final_args["response_format"] = response_format
+        return final_args
+
+    def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> list[ChatMessage]:
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+        for chunk in chat_completion:
+            assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
+            chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
+                chunk=chunk, previous_chunks=chunks, component_info=component_info
+            )
+            chunks.append(chunk_delta)
+            callback(chunk_delta)
+        return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
+
+    async def _handle_async_stream_response(
+        self, chat_completion: AsyncStream, callback: StreamingCallbackT
+    ) -> list[ChatMessage]:
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+        try:
+            async for chunk in chat_completion:
+                assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
+                chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
+                    chunk=chunk, previous_chunks=chunks, component_info=component_info
+                )
+                chunks.append(chunk_delta)
+                await _invoke_streaming_callback(callback, chunk_delta)
+
+        except asyncio.CancelledError:
+            await asyncio.shield(chat_completion.close())
+            # close the stream when task is cancelled
+            # asyncio.shield ensures the close operation completes
+            # https://docs.python.org/3/library/asyncio-task.html#shielding-from-cancellation
+            raise  # Re-raise to propagate cancellation
+
+        return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
+
+
+def _make_schema_strict(schema: dict[str, Any]) -> dict[str, Any]:
+    """
+    Recursively transform a JSON schema to be OpenAI strict-mode compliant.
+
+    Sets `additionalProperties: false` on all objects and ensures every defined
+    property is listed in `required`. Walks into nested properties, `$defs`,
+    array `items`, and `anyOf`/`oneOf`/`allOf` combinators.
+
+    See https://platform.openai.com/docs/guides/structured-outputs#supported-schemas
+    """
+    schema = {**schema}
+
+    schema_type = schema.get("type")
+
+    if schema_type == "object" or "properties" in schema:
+        schema["additionalProperties"] = False
+        if "properties" in schema:
+            schema["required"] = list(schema["properties"].keys())
+            schema["properties"] = {k: _make_schema_strict(v) for k, v in schema["properties"].items()}
+
+    if "items" in schema:
+        schema["items"] = _make_schema_strict(schema["items"])
+
+    if "$defs" in schema:
+        schema["$defs"] = {k: _make_schema_strict(v) for k, v in schema["$defs"].items()}
+
+    for combinator in ("anyOf", "oneOf", "allOf"):
+        if combinator in schema:
+            schema[combinator] = [_make_schema_strict(s) for s in schema[combinator]]
+
+    return schema
+
+
+def _check_finish_reason(meta: dict[str, Any]) -> None:
+    if meta["finish_reason"] == "length":
+        logger.warning(
+            "The completion for index {index} has been truncated before reaching a natural stopping point. "
+            "Increase the max_completion_tokens parameter to allow for longer completions.",
+            index=meta["index"],
+            finish_reason=meta["finish_reason"],
+        )
+    if meta["finish_reason"] == "content_filter":
+        logger.warning(
+            "The completion for index {index} has been truncated due to the content filter.",
+            index=meta["index"],
+            finish_reason=meta["finish_reason"],
+        )
+
+
+def _convert_chat_completion_to_chat_message(
+    completion: ChatCompletion | ParsedChatCompletion, choice: Choice
+) -> ChatMessage:
+    """
+    Converts the non-streaming response from the OpenAI API to a ChatMessage.
+
+    :param completion: The completion returned by the OpenAI API.
+    :param choice: The choice returned by the OpenAI API.
+    :return: The ChatMessage.
+    """
+    message: ChatCompletionMessage | ParsedChatCompletionMessage = choice.message
+    text = message.content
+    tool_calls = []
+    if message.tool_calls:
+        # we currently only support function tools (not custom tools)
+        # https://platform.openai.com/docs/guides/function-calling#custom-tools
+        openai_tool_calls = [tc for tc in message.tool_calls if not isinstance(tc, ChatCompletionMessageCustomToolCall)]
+        for openai_tc in openai_tool_calls:
+            arguments_str = openai_tc.function.arguments
+            try:
+                arguments = json.loads(arguments_str)
+                tool_calls.append(ToolCall(id=openai_tc.id, tool_name=openai_tc.function.name, arguments=arguments))
+            except json.JSONDecodeError:
+                logger.warning(
+                    "OpenAI returned a malformed JSON string for tool call arguments. This tool call "
+                    "will be skipped. To always generate a valid JSON, set `tools_strict` to `True`. "
+                    "Tool call ID: {_id}, Tool name: {_name}, Arguments: {_arguments}",
+                    _id=openai_tc.id,
+                    _name=openai_tc.function.name,
+                    _arguments=arguments_str,
+                )
+
+    logprobs = _serialize_object(choice.logprobs) if choice.logprobs else None
+    meta = {
+        "model": completion.model,
+        "index": choice.index,
+        "finish_reason": choice.finish_reason,
+        "usage": _serialize_object(completion.usage),
+    }
+    if logprobs:
+        meta["logprobs"] = logprobs
+
+    return ChatMessage.from_assistant(text=text, tool_calls=tool_calls, meta=meta)
+
+
+def _convert_chat_completion_chunk_to_streaming_chunk(
+    chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: ComponentInfo | None = None
+) -> StreamingChunk:
+    """
+    Converts the streaming response chunk from the OpenAI API to a StreamingChunk.
+
+    :param chunk: The chunk returned by the OpenAI API.
+    :param previous_chunks: A list of previously received StreamingChunks.
+    :param component_info: An optional `ComponentInfo` object containing information about the component that
+        generated the chunk, such as the component name and type.
+
+    :returns:
+        A StreamingChunk object representing the content of the chunk from the OpenAI API.
+    """
+    finish_reason_mapping: dict[str, FinishReason] = {
+        "stop": "stop",
+        "length": "length",
+        "content_filter": "content_filter",
+        "tool_calls": "tool_calls",
+        "function_call": "tool_calls",
+    }
+    # On very first chunk so len(previous_chunks) == 0, the Choices field only provides role info (e.g. "assistant")
+    # Choices is empty if include_usage is set to True where the usage information is returned.
+    if len(chunk.choices) == 0:
+        return StreamingChunk(
+            content="",
+            component_info=component_info,
+            # Index is None since it's only set to an int when a content block is present
+            index=None,
+            finish_reason=None,
+            meta={
+                "model": chunk.model,
+                "received_at": datetime.now().isoformat(),
+                "usage": _serialize_object(chunk.usage),
+            },
+        )
+
+    choice: ChunkChoice = chunk.choices[0]
+
+    # create a list of ToolCallDelta objects from the tool calls
+    if choice.delta and choice.delta.tool_calls:
+        tool_calls_deltas = []
+        for tool_call in choice.delta.tool_calls:
+            function = tool_call.function
+            tool_calls_deltas.append(
+                ToolCallDelta(
+                    index=tool_call.index,
+                    id=tool_call.id,
+                    tool_name=function.name if function else None,
+                    arguments=function.arguments if function and function.arguments else None,
+                )
+            )
+        return StreamingChunk(
+            content=choice.delta.content or "",
+            component_info=component_info,
+            # We adopt the first tool_calls_deltas.index as the overall index of the chunk.
+            index=tool_calls_deltas[0].index,
+            tool_calls=tool_calls_deltas,
+            start=tool_calls_deltas[0].tool_name is not None,
+            finish_reason=finish_reason_mapping.get(choice.finish_reason) if choice.finish_reason else None,
+            meta={
+                "model": chunk.model,
+                "index": choice.index,
+                "tool_calls": choice.delta.tool_calls,
+                "finish_reason": choice.finish_reason,
+                "received_at": datetime.now().isoformat(),
+                "usage": _serialize_object(chunk.usage),
+            },
+        )
+
+    # On very first chunk the choice field only provides role info (e.g. "assistant") so we set index to None
+    # We set all chunks missing the content field to index of None. E.g. can happen if chunk only contains finish
+    # reason.
+    if choice.delta and (choice.delta.content is None or choice.delta.role is not None):
+        resolved_index = None
+    else:
+        # We set the index to be 0 since if text content is being streamed then no tool calls are being streamed
+        # NOTE: We may need to revisit this if OpenAI allows planning/thinking content before tool calls like
+        #       Anthropic Claude
+        resolved_index = 0
+
+    # Initialize meta dictionary
+    meta = {
+        "model": chunk.model,
+        "index": choice.index,
+        "tool_calls": choice.delta.tool_calls if choice.delta and choice.delta.tool_calls else None,
+        "finish_reason": choice.finish_reason,
+        "received_at": datetime.now().isoformat(),
+        "usage": _serialize_object(chunk.usage),
+    }
+
+    # check if logprobs are present
+    # logprobs are returned only for text content
+    logprobs = _serialize_object(choice.logprobs) if choice.logprobs else None
+    if logprobs:
+        meta["logprobs"] = logprobs
+
+    content = ""
+    if choice.delta and choice.delta.content:
+        content = choice.delta.content
+
+    return StreamingChunk(
+        content=content,
+        component_info=component_info,
+        index=resolved_index,
+        # The first chunk is always a start message chunk that only contains role information, so if we reach here
+        # and previous_chunks is length 1 then this is the start of text content.
+        start=len(previous_chunks) == 1,
+        finish_reason=finish_reason_mapping.get(choice.finish_reason) if choice.finish_reason else None,
+        meta=meta,
+    )

haystack/components/generators/chat/types/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .protocol import ChatGenerator
+
+__all__ = ["ChatGenerator"]

haystack/components/generators/chat/types/protocol.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Protocol
+
+from haystack.dataclasses import ChatMessage
+
+
+class ChatGenerator(Protocol):
+    """
+    Protocol for Chat Generators.
+
+    This protocol defines the minimal interface that Chat Generators must implement.
+    Chat Generators are components that process a list of `ChatMessage` objects as input and generate
+    responses using a Language Model. They return a dictionary.
+    """
+
+    def run(self, messages: list[ChatMessage]) -> dict[str, Any]:
+        """
+        Generate messages using the underlying Language Model.
+
+        Implementing classes may accept additional optional parameters in their run method.
+        For example: `def run (self, messages: list[ChatMessage], param_a="default", param_b="another_default")`.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages.
+        :returns:
+            A dictionary.
+        """
+        ...