chore: import upstream snapshot with attribution

haystack/__init__.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# We avoid lazy imports here because:
+# - they create potential static type checking issues which are hard to debug
+# - they make this module more complicated and hard to maintain
+# - they offer minimal performance gains in this case.
+
+import haystack.logging
+
+# Imported so the `haystack.tracing` namespace is available after `import haystack`.
+import haystack.tracing  # noqa: F401
+from haystack.core.component import component
+from haystack.core.errors import ComponentError, DeserializationError
+from haystack.core.pipeline import Pipeline
+from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.core.super_component.super_component import SuperComponent, super_component
+from haystack.dataclasses import Answer, Document, ExtractedAnswer, GeneratedAnswer
+from haystack.version import __version__  # noqa: F401
+
+# Initialize the logging configuration.
+# This is a no-op unless `structlog` is installed. `configure_structlog=False` means we only install our own scoped
+# logging handler (so Haystack's logs are formatted) without touching the process-global `structlog` configuration.
+haystack.logging.configure_logging(configure_structlog=False)
+
+__all__ = [
+    "Answer",
+    "ComponentError",
+    "DeserializationError",
+    "Document",
+    "ExtractedAnswer",
+    "GeneratedAnswer",
+    "Pipeline",
+    "SuperComponent",
+    "super_component",
+    "component",
+    "default_from_dict",
+    "default_to_dict",
+]

haystack/components/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

haystack/components/agents/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"agent": ["Agent"], "state": ["State"]}
+
+if TYPE_CHECKING:
+    from .agent import Agent as Agent
+    from .state import State as State
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/agents/state/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"state": ["State", "merge_lists", "replace_values"]}
+
+if TYPE_CHECKING:
+    from .state import State as State
+    from .state_utils import merge_lists as merge_lists
+    from .state_utils import replace_values as replace_values
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/agents/state/state.py

@@ -0,0 +1,207 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from copy import deepcopy
+from typing import Any, get_args
+
+from haystack.dataclasses import ChatMessage
+from haystack.utils import _deserialize_value_with_schema, _serialize_value_with_schema
+from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
+from haystack.utils.type_serialization import deserialize_type, serialize_type
+
+from .state_utils import _is_list_type, _is_valid_type, merge_lists, replace_values
+
+
+def _schema_to_dict(schema: dict[str, Any]) -> dict[str, Any]:
+    """
+    Convert a schema dictionary to a serializable format.
+
+    Converts each parameter's type and optional handler function into a serializable
+    format using type and callable serialization utilities.
+
+    :param schema: Dictionary mapping parameter names to their type and handler configs
+    :returns: Dictionary with serialized type and handler information
+    """
+    serialized_schema = {}
+    for param, config in schema.items():
+        serialized_schema[param] = {"type": serialize_type(config["type"])}
+        if config.get("handler"):
+            serialized_schema[param]["handler"] = serialize_callable(config["handler"])
+
+    return serialized_schema
+
+
+def _schema_from_dict(schema: dict[str, Any]) -> dict[str, Any]:
+    """
+    Convert a serialized schema dictionary back to its original format.
+
+    Deserializes the type and optional handler function for each parameter from their
+    serialized format back into Python types and callables.
+
+    :param schema: Dictionary containing serialized schema information
+    :returns: Dictionary with deserialized type and handler configurations
+    """
+    deserialized_schema = {}
+    for param, config in schema.items():
+        deserialized_schema[param] = {"type": deserialize_type(config["type"])}
+
+        if config.get("handler"):
+            deserialized_schema[param]["handler"] = deserialize_callable(config["handler"])
+
+    return deserialized_schema
+
+
+def _validate_schema(schema: dict[str, Any]) -> None:
+    """
+    Validate that a schema dictionary meets all required constraints.
+
+    Checks that each parameter definition has a valid type field and that any handler
+    specified is a callable function.
+
+    :param schema: Dictionary mapping parameter names to their type and handler configs
+    :raises ValueError: If schema validation fails due to missing or invalid fields
+    """
+    for param, definition in schema.items():
+        if "type" not in definition:
+            raise ValueError(f"StateSchema: Key '{param}' is missing a 'type' entry.")
+        if not _is_valid_type(definition["type"]):
+            raise ValueError(f"StateSchema: 'type' for key '{param}' must be a Python type, got {definition['type']}")
+        if definition.get("handler") is not None and not callable(definition["handler"]):
+            raise ValueError(f"StateSchema: 'handler' for key '{param}' must be callable or None")
+        if param == "messages":  # definition["type"] != list[ChatMessage] but split to cover also List[ChatMessage]
+            if not _is_list_type(definition["type"]):
+                raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
+            # Check if the list contains ChatMessage elements
+            args = get_args(definition["type"])
+            if not args or not issubclass(args[0], ChatMessage):
+                raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
+
+
+class State:
+    """
+    State is a container for storing shared information during the execution of an Agent and its tools.
+
+    For instance, State can be used to store documents, context, and intermediate results.
+
+    Internally it wraps a `_data` dictionary defined by a `schema`. Each schema entry has:
+    ```json
+      "parameter_name": {
+        "type": SomeType,  # expected type
+        "handler": Optional[Callable[[Any, Any], Any]]  # merge/update function
+      }
+      ```
+
+    Handlers control how values are merged when using the `set()` method:
+    - For list types: defaults to `merge_lists` (concatenates lists)
+    - For other types: defaults to `replace_values` (overwrites existing value)
+
+    A `messages` field with type `list[ChatMessage]` is automatically added to the schema.
+
+    This makes it possible for the Agent to read from and write to the same context.
+
+    ### Usage example
+    ```python
+    from haystack.components.agents.state import State
+
+    my_state = State(
+        schema={"gh_repo_name": {"type": str}, "user_name": {"type": str}},
+        data={"gh_repo_name": "my_repo", "user_name": "my_user_name"}
+    )
+    ```
+    """
+
+    def __init__(self, schema: dict[str, Any], data: dict[str, Any] | None = None) -> None:
+        """
+        Initialize a State object with a schema and optional data.
+
+        :param schema: Dictionary mapping parameter names to their type and handler configs.
+            Type must be a valid Python type, and handler must be a callable function or None.
+            If handler is None, the default handler for the type will be used. The default handlers are:
+                - For list types: `haystack.agents.state.state_utils.merge_lists`
+                - For all other types: `haystack.agents.state.state_utils.replace_values`
+        :param data: Optional dictionary of initial data to populate the state
+        """
+        _validate_schema(schema)
+        self.schema = deepcopy(schema)
+        if self.schema.get("messages") is None:
+            self.schema["messages"] = {"type": list[ChatMessage], "handler": merge_lists}
+        self._data = data or {}
+
+        # Set default handlers if not provided in schema
+        for definition in self.schema.values():
+            # Skip if handler is already defined and not None
+            if definition.get("handler") is not None:
+                continue
+            # Set default handler based on type
+            if _is_list_type(definition["type"]):
+                definition["handler"] = merge_lists
+            else:
+                definition["handler"] = replace_values
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Retrieve a value from the state by key.
+
+        :param key: Key to look up in the state
+        :param default: Value to return if key is not found
+        :returns: Value associated with key or default if not found
+        """
+        return deepcopy(self._data.get(key, default))
+
+    def set(self, key: str, value: Any, handler_override: Callable[[Any, Any], Any] | None = None) -> None:
+        """
+        Set or merge a value in the state according to schema rules.
+
+        Value is merged or overwritten according to these rules:
+          - if handler_override is given, use that
+          - else use the handler defined in the schema for 'key'
+
+        :param key: Key to store the value under
+        :param value: Value to store or merge
+        :param handler_override: Optional function to override the default merge behavior
+        """
+        # If key not in schema, we throw an error
+        definition = self.schema.get(key, None)
+        if definition is None:
+            raise ValueError(f"State: Key '{key}' not found in schema. Schema: {self.schema}")
+
+        # Get current value from state and apply handler
+        current_value = self._data.get(key, None)
+        handler = handler_override or definition["handler"]
+        self._data[key] = handler(current_value, value)
+
+    @property
+    def data(self) -> dict[str, Any]:
+        """
+        All current data of the state.
+        """
+        return self._data
+
+    def has(self, key: str) -> bool:
+        """
+        Check if a key exists in the state.
+
+        :param key: Key to check for existence
+        :returns: True if key exists in state, False otherwise
+        """
+        return key in self._data
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Convert the State object to a dictionary.
+        """
+        serialized = {}
+        serialized["schema"] = _schema_to_dict(self.schema)
+        serialized["data"] = _serialize_value_with_schema(self._data)
+        return serialized
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "State":
+        """
+        Convert a dictionary back to a State object.
+        """
+        schema = _schema_from_dict(data.get("schema", {}))
+        deserialized_data = _deserialize_value_with_schema(data.get("data", {}))
+        return State(schema, deserialized_data)

haystack/components/agents/state/state_utils.py

@@ -0,0 +1,86 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import inspect
+from typing import Any, TypeVar, Union, get_origin
+
+from haystack.utils.type_serialization import _is_union_type
+
+T = TypeVar("T")
+
+
+def _is_valid_type(obj: Any) -> bool:
+    """
+    Check if an object is a valid type annotation.
+
+    Valid types include:
+    - Normal classes (str, dict, CustomClass)
+    - Generic types (list[str], dict[str, int])
+    - Union types (Union[str, int], Optional[str], str | int, str | None)
+
+    :param obj: The object to check
+    :return: True if the object is a valid type annotation, False otherwise
+
+    Example usage:
+        # >> _is_valid_type(str)
+        # >> True
+        # >> _is_valid_type(list[int])
+        # >> True
+        # >> _is_valid_type(Union[str, int])
+        # >> True
+        # >> _is_valid_type(str | int)
+        # >> True
+        # >> _is_valid_type(42)
+        # >> False
+    """
+    # Handle Union types (including Optional)
+    if (origin := get_origin(obj)) and _is_union_type(origin):
+        return True
+
+    # Bare Union type (without parameters) is not a valid type annotation
+    # Previously handled by inspect.isclass(obj) but in python 3.14 this returns True for typing.Union
+    if obj == Union:
+        return False
+
+    # Handle normal classes and generic types
+    return inspect.isclass(obj) or type(obj).__name__ in {"_GenericAlias", "GenericAlias"}
+
+
+def _is_list_type(type_hint: Any) -> bool:
+    """
+    Check if a type hint represents a list type.
+
+    :param type_hint: The type hint to check
+    :return: True if the type hint represents a list, False otherwise
+    """
+    return type_hint == list or (hasattr(type_hint, "__origin__") and get_origin(type_hint) == list)
+
+
+def merge_lists(current: Union[list[T], T, None], new: Union[list[T], T]) -> list[T]:
+    """
+    Merges two values into a single list.
+
+    If either `current` or `new` is not already a list, it is converted into one.
+    The function ensures that both inputs are treated as lists and concatenates them.
+
+    If `current` is None, it is treated as an empty list.
+
+    :param current: The existing value(s), either a single item or a list.
+    :param new: The new value(s) to merge, either a single item or a list.
+    :return: A list containing elements from both `current` and `new`.
+    """
+    current_list = [] if current is None else current if isinstance(current, list) else [current]
+    new_list = new if isinstance(new, list) else [new]
+    return current_list + new_list
+
+
+def replace_values(current: Any, new: Any) -> Any:  # noqa: ARG001
+    """
+    Replace the `current` value with the `new` value.
+
+    :param current: The existing value
+    :param new: The new value to replace
+    :return: The new value
+    """
+    return new

haystack/components/agents/tool_calling.py

@@ -0,0 +1,667 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import contextvars
+import inspect
+import json
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from haystack import logging, tracing
+from haystack.components.agents.state.state import State
+from haystack.core.component.sockets import Sockets
+from haystack.dataclasses import ChatMessage, ToolCall
+from haystack.dataclasses.streaming_chunk import StreamingCallbackT, StreamingChunk, _invoke_streaming_callback
+from haystack.tools import ComponentTool, Tool, ToolsType, _check_duplicate_tool_names, flatten_tools_or_toolsets
+from haystack.tools.errors import ToolInvocationError
+from haystack.tools.parameters_schema_utils import _unwrap_optional
+from haystack.tracing.utils import _serializable_value
+
+logger = logging.getLogger(__name__)
+
+
+class _AllStateKeys:
+    """Sentinel representing "every state key", used for tools that receive the full State object."""
+
+    def __repr__(self) -> str:
+        return "<all state keys>"
+
+
+_ALL_STATE_KEYS = _AllStateKeys()
+
+# A set of state keys, or the _ALL_STATE_KEYS sentinel meaning "every key".
+_StateKeys = set[str] | _AllStateKeys
+
+
+class ToolNotFoundException(Exception):
+    """Exception raised when a tool is not found in the list of available tools."""
+
+    def __init__(self, tool_name: str, available_tools: list[str]) -> None:
+        message = f"Tool '{tool_name}' not found. Available tools: {', '.join(available_tools)}"
+        super().__init__(message)
+
+
+def _validate_and_prepare_tools(tools: ToolsType) -> dict[str, Tool]:
+    """
+    Flatten, deduplicate-check, and index tools by name.
+
+    :raises ValueError: If no tools are provided or if duplicate tool names are found.
+    """
+    if not tools:
+        raise ValueError("Tool execution requires at least one tool.")
+
+    available_tools = flatten_tools_or_toolsets(tools)
+    _check_duplicate_tool_names(available_tools)
+    tool_names = [tool.name for tool in available_tools]
+
+    return dict(zip(tool_names, available_tools, strict=True))
+
+
+def _merge_tool_outputs_into_state(tool: Tool, result: Any, state: State) -> None:
+    """
+    Write tool outputs into State according to the tool's `outputs_to_state` mapping.
+
+    :raises RuntimeError: If writing an output value into the state fails.
+    """
+    if not isinstance(result, dict):
+        return
+
+    for state_key, config in (tool.outputs_to_state or {}).items():
+        source_key = config.get("source", None)
+        if source_key and source_key not in result:
+            continue
+        output_value = result.get(source_key) if source_key else result
+        try:
+            state.set(state_key, output_value, handler_override=config.get("handler"))
+        except Exception as e:
+            raise RuntimeError(f"Tool '{tool.name}': failed to merge outputs into state. {e}") from e
+
+
+def _result_to_string(result: Any) -> str:
+    """
+    Convert a tool result to a string.
+
+    Strings are returned as-is; all other types are passed through a JSON serialization step to produce more readable
+    output, with a fallback to plain str() conversion if serialization fails.
+
+    :param result: The tool result to convert.
+    :returns: A string representation of the tool result.
+    """
+    if isinstance(result, str):
+        return result
+    serializable = _serializable_value(value=result, use_placeholders=False)
+    try:
+        return json.dumps(serializable, ensure_ascii=False)
+    except Exception as error:
+        logger.warning(
+            "Tool result is not JSON serializable. Falling back to str conversion. Result: {result}\nError: {err}",
+            result=result,
+            err=error,
+        )
+        return str(result)
+
+
+def _process_tool_output(config: dict[str, Any], result: Any, tool_call: ToolCall, *, raise_on_failure: bool) -> Any:
+    """
+    Extract and convert a single tool output according to `config`.
+
+    `config` may contain `source` (key to extract from result dict), `handler` (conversion callable), and
+    `raw_result` (return the value without string conversion).
+
+    If a configured `handler` raises, the exception is re-raised when `raise_on_failure` is True; otherwise
+    a warning is logged and the value is converted via `_result_to_string`.
+    """
+    source_key = config.get("source")
+    value = result.get(source_key) if source_key is not None and isinstance(result, dict) else result
+
+    handler = config.get("handler")
+    raw_result = config.get("raw_result", False)
+
+    if handler is None:
+        # raw result is mostly used to allow ImageContent or TextContent blocks to be directly returned and consumed
+        # by ChatMessage.from_tool without string conversion.
+        if raw_result:
+            return value
+        return _result_to_string(value)
+
+    try:
+        return handler(value)
+    except Exception as e:
+        if raise_on_failure:
+            raise
+        logger.warning(
+            "Output handler '{handler}' for tool '{tool}' failed, falling back to string conversion. Error: {err}",
+            handler=handler.__name__,
+            tool=tool_call.tool_name,
+            err=e,
+        )
+        return _result_to_string(value)
+
+
+def _build_tool_result_message(result: Any, tool_call: ToolCall, tool: Tool, *, raise_on_failure: bool) -> ChatMessage:
+    """Convert a raw tool result into a ChatMessage, applying `outputs_to_string` config if present."""
+    outputs_config = tool.outputs_to_string or {}
+
+    # Single-output config (or no config): keys are at the root level
+    if not outputs_config or any(k in outputs_config for k in ("source", "handler", "raw_result")):
+        tool_result = _process_tool_output(outputs_config, result, tool_call, raise_on_failure=raise_on_failure)
+        return ChatMessage.from_tool(tool_result=tool_result, origin=tool_call)
+
+    # Multi-output config: each key maps to its own sub-config — stringify each value, then stringify the whole dict
+    tool_result_dict = {
+        output_key: _process_tool_output(
+            {**cfg, "raw_result": False}, result, tool_call, raise_on_failure=raise_on_failure
+        )
+        for output_key, cfg in outputs_config.items()
+    }
+    return ChatMessage.from_tool(tool_result=_result_to_string(tool_result_dict), origin=tool_call)
+
+
+def _create_tool_result_streaming_chunk(tool_message: ChatMessage, tool_call: ToolCall, index: int) -> StreamingChunk:
+    """
+    Create a streaming chunk that carries a tool result.
+
+    :param tool_message: The tool result message to stream.
+    :param tool_call: The ToolCall object that triggered the tool invocation.
+    :param index: The position of this tool result in the stream (in execution order).
+    :returns: A StreamingChunk containing the tool result and metadata about the tool call.
+    """
+    return StreamingChunk(
+        content="",
+        index=index,
+        tool_call_result=tool_message.tool_call_results[0],
+        start=True,
+        meta={"tool_result": tool_message.tool_call_results[0].result, "tool_call": tool_call},
+    )
+
+
+def _create_tool_span(tool: Tool, tool_call: ToolCall) -> Any:
+    """
+    Create one tracing span for a single tool call, nested under the currently active span.
+
+    The established standard for tracing agents is one span per tool call, so each call gets its own span rather than
+    grouping all of a step's calls together. The OpenTelemetry GenAI semantic conventions codify this with a dedicated
+    "execute tool" span per invocation
+    (https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#execute-tool-span),
+    and tracing backends such as Langfuse follow the same model. The span is tagged with the tool's identity; the caller
+    adds the call arguments and result as content tags.
+    """
+    return tracing.tracer.trace(
+        "haystack.agent.step.tool",
+        tags={"haystack.tool.name": tool_call.tool_name, "haystack.tool.description": tool.description},
+        parent_span=tracing.tracer.current_span(),
+    )
+
+
+def _make_context_bound_invoke(tool: Tool, args: dict[str, Any], tool_call: ToolCall) -> Callable[[], Any]:
+    """
+    Return a zero-arg callable that runs `tool.invoke(**args)` under the current contextvars snapshot.
+
+    This preserves tracing spans and other context-local state across thread-pool boundaries, so the per-call span
+    created inside the worker nests correctly under the step span. The callable returns a ToolInvocationError instead
+    of raising so that parallel executions can collect failures without aborting the whole batch.
+    """
+    ctx = contextvars.copy_context()
+
+    def _invoke() -> Any:
+        with _create_tool_span(tool, tool_call) as span:
+            span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments)
+            try:
+                result = tool.invoke(**args)
+            except ToolInvocationError as e:
+                span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
+                return e
+            span.set_content_tag("haystack.agent.step.tool.output", result)
+            return result
+
+    def _runner() -> Any:
+        return ctx.run(_invoke)
+
+    return _runner
+
+
+def _make_bounded_invoke_async(
+    tool: Tool, args: dict[str, Any], semaphore: asyncio.Semaphore, tool_call: ToolCall
+) -> Callable[[], Any]:
+    """
+    Return a zero-arg async callable that awaits `tool.invoke_async(**args)` while holding `semaphore`.
+
+    Concurrency is bounded uniformly across native-async tools and sync-fallback tools (which dispatch
+    to a worker thread inside `Tool.invoke_async`). ContextVars naturally inherit into child tasks for
+    the native-async branch, and `asyncio.to_thread` propagates them for the fallback branch, so the per-call
+    span nests correctly under the step span.
+
+    Returns a `ToolInvocationError` instead of raising so that gathered executions can collect failures
+    without aborting the whole batch.
+    """
+
+    async def _runner() -> Any:
+        async with semaphore:
+            with _create_tool_span(tool, tool_call) as span:
+                span.set_content_tag("haystack.agent.step.tool.input", tool_call.arguments)
+                try:
+                    result = await tool.invoke_async(**args)
+                except ToolInvocationError as e:
+                    span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
+                    return e
+                span.set_content_tag("haystack.agent.step.tool.output", result)
+                return result
+
+    return _runner
+
+
+def _get_func_params(tool: Tool) -> dict[str, Any]:
+    """
+    Return parameter names → annotations for a tool's invocation function.
+
+    - For ComponentTool, this is the annotated input schema defined on the underlying component.
+    - For regular Tools, this is the function signature of the `function` callable, falling back to `async_function`
+      for async-only tools.
+
+    :param tool: The tool to inspect.
+    :returns: A dict mapping parameter names to their type annotations.
+    """
+    if isinstance(tool, ComponentTool):
+        assert hasattr(tool._component, "__haystack_input__") and isinstance(
+            tool._component.__haystack_input__, Sockets
+        )
+        return {name: socket.type for name, socket in tool._component.__haystack_input__._sockets_dict.items()}
+    # Tool.__post_init__ guarantees that at least one of `function` / `async_function` is set.
+    target = tool.function if tool.function is not None else tool.async_function
+    return {name: param.annotation for name, param in inspect.signature(target).parameters.items()}  # type: ignore[arg-type]
+
+
+def _inject_state_args(tool: Tool, llm_args: dict[str, Any], state: State) -> dict[str, Any]:
+    """
+    Merge LLM-provided arguments with state-sourced arguments.
+
+    LLM args take precedence. State values are pulled in only for the keys a tool explicitly declares via its
+    `inputs_from_state` mapping, then the live State object is injected for any param annotated as State.
+
+    :param tool: The tool being invoked, used to determine parameter mappings and State injection.
+    :param llm_args: The arguments provided by the LLM, which take precedence over state values.
+    :param state: The current runtime state, used to source additional arguments as needed.
+    :returns: A dict of arguments to invoke the tool with, combining LLM and state values according to the rules
+        described above.
+    """
+    final_args = dict(llm_args)
+    func_params = _get_func_params(tool)
+
+    # A tool reads from State by name only via an explicit `inputs_from_state` mapping
+    for state_key, param_name in (tool.inputs_from_state or {}).items():
+        if param_name not in final_args and state.has(state_key):
+            final_args[param_name] = state.get(state_key)
+
+    # We also inject the full State object for any parameter annotated as State
+    for param_name, param_type in func_params.items():
+        if _unwrap_optional(param_type) is State:
+            final_args[param_name] = state
+
+    return final_args
+
+
+def _prepare_tool_args(
+    *,
+    tool: Tool,
+    tool_call_arguments: dict[str, Any],
+    state: State,
+    streaming_callback: StreamingCallbackT | None = None,
+    enable_streaming_passthrough: bool = False,
+) -> dict[str, Any]:
+    """
+    Prepare the final arguments for a tool by injecting state inputs and optionally a streaming callback.
+
+    :param tool:
+        The tool instance to prepare arguments for.
+    :param tool_call_arguments:
+        The initial arguments provided for the tool call.
+    :param state:
+        The current state containing inputs to be injected into the tool arguments.
+    :param streaming_callback:
+        Optional streaming callback to be injected if enabled and applicable.
+    :param enable_streaming_passthrough:
+        Flag indicating whether to inject the streaming callback into the tool arguments.
+
+    :returns:
+        A dictionary of final arguments ready for tool invocation.
+    """
+    # Combine user + state inputs
+    final_args = _inject_state_args(tool, tool_call_arguments.copy(), state)
+    # Check whether to inject streaming_callback
+    if (
+        enable_streaming_passthrough
+        and streaming_callback is not None
+        and "streaming_callback" not in final_args
+        and "streaming_callback" in _get_func_params(tool)
+    ):
+        final_args["streaming_callback"] = streaming_callback
+    return final_args
+
+
+def _resolve_tool_calls(
+    messages_with_tool_calls: list[ChatMessage], tools_with_names: dict[str, Tool], *, raise_on_failure: bool
+) -> tuple[list[ToolCall], list[Tool], list[ChatMessage]]:
+    """
+    Walk all tool calls in `messages_with_tool_calls` and resolve each to its Tool.
+
+    Argument preparation is deliberately *not* done here: args are prepared per execution batch (see
+    `_schedule_tool_calls`) so that a tool reading from State observes writes made by tools that ran earlier in the same
+     step.
+
+    :returns: (tool_calls, resolved_tools, error_messages)
+        - tool_calls: ToolCall objects for each valid call, in call order
+        - resolved_tools: the resolved Tool for each entry in `tool_calls` (parallel list)
+        - error_messages: ChatMessages for tool-not-found errors (when raise_on_failure is False)
+    """
+    tool_calls: list[ToolCall] = []
+    resolved_tools: list[Tool] = []
+    error_messages: list[ChatMessage] = []
+
+    for message in messages_with_tool_calls:
+        for tool_call in message.tool_calls:
+            tool_name = tool_call.tool_name
+
+            if tool_name not in tools_with_names:
+                error = ToolNotFoundException(tool_name, list(tools_with_names.keys()))
+                if raise_on_failure:
+                    raise error
+                logger.error("{error_exception}", error_exception=error)
+                error_messages.append(ChatMessage.from_tool(tool_result=str(error), origin=tool_call, error=True))
+                continue
+
+            tool_calls.append(tool_call)
+            resolved_tools.append(tools_with_names[tool_name])
+
+    return tool_calls, resolved_tools, error_messages
+
+
+def _keys_intersect(a: _StateKeys, b: _StateKeys) -> bool:
+    """
+    Return whether two State-key sets share at least one key, treating `_ALL_STATE_KEYS` as a wildcard.
+
+    Used to detect read-after-write dependencies between tool calls: the reader's read set is tested against the
+    writer's write set.
+
+    :param a: A set of state keys, or the `_ALL_STATE_KEYS` wildcard meaning "every key".
+    :param b: A set of state keys, or the `_ALL_STATE_KEYS` wildcard meaning "every key".
+    :returns: True if the sets overlap (a wildcard overlaps any non-empty set, and two wildcards always overlap).
+    """
+    if a is _ALL_STATE_KEYS:
+        # `a` covers every key, so it overlaps `b` as long as `b` touches any key. Two wildcards always overlap;
+        # otherwise `bool(b)` is True iff the concrete set `b` is non-empty.
+        return b is _ALL_STATE_KEYS or bool(b)
+    if b is _ALL_STATE_KEYS:
+        # Symmetric case: wildcard `b` overlaps `a` iff the concrete set `a` is non-empty (`bool(set)` == non-empty).
+        return bool(a)
+    # Both are concrete sets: they overlap iff their set intersection is non-empty.
+    return bool(a & b)  # type: ignore[operator]
+
+
+def _state_io_for_call(tool: Tool, llm_args: dict[str, Any]) -> tuple[_StateKeys, _StateKeys]:
+    """
+    Compute the State keys a tool call reads from and writes to.
+
+    Mirrors the resolution logic in `_inject_state_args`:
+    - A tool with a `State`-annotated parameter can read/write any key, so both sets are the `_ALL_STATE_KEYS` wildcard.
+    - Otherwise reads come from the tool's explicit `inputs_from_state` mapping, excluding any parameter the LLM already
+      supplied (LLM args take precedence and short-circuit the state lookup).
+    - Writes are the keys in `outputs_to_state`.
+
+    :returns: A `(reads, writes)` tuple of state-key sets (or the `_ALL_STATE_KEYS` wildcard).
+    """
+    func_params = _get_func_params(tool)
+    # Check if State is in func_params
+    if any(_unwrap_optional(param_type) is State for param_type in func_params.values()):
+        return _ALL_STATE_KEYS, _ALL_STATE_KEYS
+
+    # Calculate reads
+    param_mappings = tool.inputs_from_state or {}
+    reads = {state_key for state_key, param_name in param_mappings.items() if param_name not in llm_args}
+    # Calculate writes
+    writes = set((tool.outputs_to_state or {}).keys())
+
+    return reads, writes
+
+
+def _schedule_tool_calls(tool_calls: list[ToolCall], tools: list[Tool]) -> list[list[int]]:
+    """
+    Group tool calls into ordered execution batches based on their State read/write sets.
+
+    Calls within a batch are mutually independent and run in parallel; batches run sequentially. The schedule guarantees
+    that a call reading a State key always runs in a later batch than any call (in the same step) that writes that
+    key — so read-after-write dependencies are honored regardless of the order the LLM requested the calls in.
+
+    This is a layered topological sort: each round, every call whose dependencies have all been scheduled forms the
+    next parallel batch. Dependency cycles — e.g. a tool that both reads and writes the same key, requested more than
+    once — cannot be ordered by the read-after-write rule alone, so they are broken deterministically by call order
+    (the lowest-index remaining call runs next, on its own).
+
+    Pure write-write overlaps create no dependency: nobody reads the contended key, and outputs are merged into State
+    sequentially in call order afterward, so the result stays deterministic without serializing execution.
+
+    :param tool_calls: The tool calls to schedule, in call order.
+    :param tools: The resolved Tool for each entry in `tool_calls` (parallel list).
+    :returns: A list of batches, each a list of indices into `tool_calls`.
+    """
+    # Per-call (reads, writes) State-key sets, in call order.
+    io_list = [_state_io_for_call(tool, tc.arguments) for tc, tool in zip(tool_calls, tools, strict=True)]
+    n = len(io_list)
+
+    # deps[j] = indices that must run before j because j reads a key they write (read-after-write).
+    deps: list[set[int]] = [set() for _ in range(n)]
+    for j in range(n):
+        reads_j, _ = io_list[j]
+        for i in range(n):
+            if i == j:
+                continue
+            _, writes_i = io_list[i]
+            if _keys_intersect(reads_j, writes_i):
+                deps[j].add(i)
+
+    scheduled = [False] * n
+    done: set[int] = set()
+    batches: list[list[int]] = []
+
+    while len(done) < n:
+        # A call is ready once every writer it depends on has already been scheduled (`deps[k] <= done`, i.e. its
+        # dependency set is a subset of the already-done set). All ready calls have no dependency on each other —
+        # if one read a key another writes, it would still be waiting — so the whole `ready` list runs in parallel.
+        ready = [k for k in range(n) if not scheduled[k] and deps[k] <= done]
+        if not ready:
+            # A dependency cycle remains: break it deterministically by running the lowest-index call next.
+            ready = [next(k for k in range(n) if not scheduled[k])]
+        for k in ready:
+            scheduled[k] = True
+        done.update(ready)
+        batches.append(ready)
+
+    return batches
+
+
+def _finalize_tool_result(
+    result: Any, tool_call: ToolCall, tool: Tool, state: State, *, raise_on_failure: bool
+) -> ChatMessage:
+    """
+    Turn a single tool invocation result into a tool-result ChatMessage, merging outputs into State.
+
+    On a `ToolInvocationError`, either re-raise (when `raise_on_failure`) or return an error message. Otherwise
+    merge the tool's outputs into State (in call order, so write-write merges stay deterministic) and build the
+    result message.
+    """
+    if isinstance(result, ToolInvocationError):
+        if raise_on_failure:
+            raise result
+        logger.error("{error_exception}", error_exception=result)
+        return ChatMessage.from_tool(tool_result=str(result), origin=tool_call, error=True)
+
+    _merge_tool_outputs_into_state(tool, result, state)
+    return _build_tool_result_message(result, tool_call, tool, raise_on_failure=raise_on_failure)
+
+
+def _run_tool(
+    *,
+    messages: list[ChatMessage],
+    state: State,
+    tools: ToolsType,
+    streaming_callback: StreamingCallbackT | None = None,
+    raise_on_failure: bool = True,
+    enable_streaming_callback_passthrough: bool = False,
+    max_workers: int = 4,
+) -> tuple[list[ChatMessage], State]:
+    """
+    Invoke all tools referenced by tool calls in `messages`.
+
+    :param messages: ChatMessage objects that may contain tool calls.
+    :param state: Runtime state passed to and updated by tools.
+    :param tools: The tools available for invocation.
+    :param streaming_callback: Called once per tool result as it becomes available.
+    :param raise_on_failure: If True, raise on tool invocation failure; otherwise return an error message.
+    :param enable_streaming_callback_passthrough: If True, pass the streaming callback to tools that accept it.
+    :param max_workers: Maximum number of parallel tool invocations.
+    :returns: (tool_messages, updated_state)
+    """
+    tools_with_names = _validate_and_prepare_tools(tools)
+
+    messages_with_tool_calls = [m for m in messages if m.tool_calls]
+    if not messages_with_tool_calls:
+        return [], state
+
+    tool_calls, resolved_tools, error_messages = _resolve_tool_calls(
+        messages_with_tool_calls, tools_with_names, raise_on_failure=raise_on_failure
+    )
+    if not tool_calls:
+        return error_messages, state
+
+    # Group the calls into batches that honor read-after-write dependencies on State (see `_schedule_tool_calls`).
+    batches = _schedule_tool_calls(tool_calls, resolved_tools)
+
+    # Results are indexed by call position so the returned messages stay in call order, even though batches may
+    # execute the calls in a different order.
+    results: list[ChatMessage | None] = [None] * len(tool_calls)
+    stream_index = 0
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for batch in batches:
+            # Prepare args at the start of each batch so tools that read from State observe writes merged by earlier
+            # batches.
+            futures = {}
+            for idx in batch:
+                args = _prepare_tool_args(
+                    tool=resolved_tools[idx],
+                    tool_call_arguments=tool_calls[idx].arguments,
+                    state=state,
+                    streaming_callback=streaming_callback,
+                    enable_streaming_passthrough=enable_streaming_callback_passthrough,
+                )
+                futures[idx] = executor.submit(_make_context_bound_invoke(resolved_tools[idx], args, tool_calls[idx]))
+
+            # Merge results in call order within the batch so write-write merges stay deterministic.
+            for idx in batch:
+                message = _finalize_tool_result(
+                    futures[idx].result(),
+                    tool_calls[idx],
+                    resolved_tools[idx],
+                    state,
+                    raise_on_failure=raise_on_failure,
+                )
+                results[idx] = message
+                if streaming_callback is not None:
+                    streaming_callback(_create_tool_result_streaming_chunk(message, tool_calls[idx], stream_index))
+                    stream_index += 1
+
+    tool_messages = error_messages + [m for m in results if m is not None]
+
+    # We emit a final empty chunk with finish_reason "tool_call_results" to signal the end of the tool results stream.
+    if tool_messages and streaming_callback is not None:
+        streaming_callback(
+            StreamingChunk(content="", finish_reason="tool_call_results", meta={"finish_reason": "tool_call_results"})
+        )
+
+    return tool_messages, state
+
+
+async def _run_tool_async(
+    *,
+    messages: list[ChatMessage],
+    state: State,
+    tools: ToolsType,
+    streaming_callback: StreamingCallbackT | None = None,
+    raise_on_failure: bool = True,
+    enable_streaming_callback_passthrough: bool = False,
+    max_workers: int = 4,
+) -> tuple[list[ChatMessage], State]:
+    """
+    Asynchronous variant of `run_tool`. Tool calls execute concurrently via a thread pool.
+
+    :param messages: ChatMessage objects that may contain tool calls.
+    :param state: Runtime state passed to and updated by tools.
+    :param tools: The tools available for invocation.
+    :param streaming_callback: Async callback called once per tool result.
+    :param raise_on_failure: If True, raise on tool invocation failure; otherwise return an error message.
+    :param enable_streaming_callback_passthrough: If True, pass the streaming callback to tools that accept it.
+    :param max_workers: Maximum number of parallel tool invocations.
+    :returns: (tool_messages, updated_state)
+    """
+    tools_with_names = _validate_and_prepare_tools(tools)
+
+    messages_with_tool_calls = [m for m in messages if m.tool_calls]
+    if not messages_with_tool_calls:
+        return [], state
+
+    tool_calls, resolved_tools, error_messages = _resolve_tool_calls(
+        messages_with_tool_calls, tools_with_names, raise_on_failure=raise_on_failure
+    )
+    if not tool_calls:
+        return error_messages, state
+
+    # Group the calls into batches that honor read-after-write dependencies on State (see `_schedule_tool_calls`).
+    batches = _schedule_tool_calls(tool_calls, resolved_tools)
+
+    # Results are indexed by call position so the returned messages stay in call order, even though batches may
+    # execute the calls in a different order.
+    results: list[ChatMessage | None] = [None] * len(tool_calls)
+    stream_index = 0
+
+    # `max_workers` + Semaphore bounds concurrency for both sync and async tool calls async tools are awaited directly,
+    # and sync tools are dispatched to a worker thread inside `Tool.invoke_async`.
+    semaphore = asyncio.Semaphore(max_workers)
+
+    for batch in batches:
+        # Prepare args at the start of each batch so readers observe writes merged by earlier batches.
+        tasks = {}
+        for idx in batch:
+            args = _prepare_tool_args(
+                tool=resolved_tools[idx],
+                tool_call_arguments=tool_calls[idx].arguments,
+                state=state,
+                streaming_callback=streaming_callback,
+                enable_streaming_passthrough=enable_streaming_callback_passthrough,
+            )
+            tasks[idx] = _make_bounded_invoke_async(resolved_tools[idx], args, semaphore, tool_calls[idx])()
+        batch_results = await asyncio.gather(*tasks.values())
+
+        # Merge results in call order within the batch so write-write merges stay deterministic.
+        for idx, result in zip(tasks.keys(), batch_results, strict=True):
+            message = _finalize_tool_result(
+                result, tool_calls[idx], resolved_tools[idx], state, raise_on_failure=raise_on_failure
+            )
+            results[idx] = message
+            if streaming_callback is not None:
+                await _invoke_streaming_callback(
+                    streaming_callback, _create_tool_result_streaming_chunk(message, tool_calls[idx], stream_index)
+                )
+                stream_index += 1
+
+    tool_messages = error_messages + [m for m in results if m is not None]
+
+    # We emit a final empty chunk with finish_reason "tool_call_results" to signal the end of the tool results stream.
+    if tool_messages and streaming_callback is not None:
+        await _invoke_streaming_callback(
+            streaming_callback,
+            StreamingChunk(content="", finish_reason="tool_call_results", meta={"finish_reason": "tool_call_results"}),
+        )
+
+    return tool_messages, state

haystack/components/builders/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "answer_builder": ["AnswerBuilder"],
+    "chat_prompt_builder": ["ChatPromptBuilder"],
+    "prompt_builder": ["PromptBuilder"],
+}
+
+if TYPE_CHECKING:
+    from .answer_builder import AnswerBuilder as AnswerBuilder
+    from .chat_prompt_builder import ChatPromptBuilder as ChatPromptBuilder
+    from .prompt_builder import PromptBuilder as PromptBuilder
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/builders/answer_builder.py

@@ -0,0 +1,314 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from dataclasses import replace
+from typing import Any
+
+from haystack import Document, GeneratedAnswer, component, logging
+from haystack.dataclasses.chat_message import ChatMessage
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_REFERENCE_PATTERN = r"\[(\d+)\]"
+EXPANDED_REFERENCE_PATTERN = r"\[(\d+(?:[,-]\d+)*)\]"
+
+
+@component
+class AnswerBuilder:
+    """
+    Converts a query and Generator replies into a `GeneratedAnswer` object.
+
+    AnswerBuilder parses Generator replies using custom regular expressions.
+    Check out the usage example below to see how it works.
+    Optionally, it can also take documents and metadata from the Generator to add to the `GeneratedAnswer` object.
+    AnswerBuilder works with both non-chat and chat Generators.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.builders import AnswerBuilder
+
+    builder = AnswerBuilder(pattern="Answer: (.*)")
+    builder.run(query="What's the answer?", replies=["This is an argument. Answer: This is the answer."])
+    ```
+
+    ### Usage example with documents and reference pattern
+
+    ```python
+    from haystack import Document
+    from haystack.components.builders import AnswerBuilder
+
+    replies = ["The capital of France is Paris [2]."]
+
+    docs = [
+        Document(content="Berlin is the capital of Germany."),
+        Document(content="Paris is the capital of France."),
+        Document(content="Rome is the capital of Italy."),
+    ]
+
+    builder = AnswerBuilder(reference_pattern="\\[(\\d+)\\]", return_only_referenced_documents=False)
+    result = builder.run(query="What is the capital of France?", replies=replies, documents=docs)["answers"][0]
+
+    print(f"Answer: {result.data}")
+    print("References:")
+    for doc in result.documents:
+        if doc.meta["referenced"]:
+            print(f"[{doc.meta['source_index']}] {doc.content}")
+    print("Other sources:")
+    for doc in result.documents:
+        if not doc.meta["referenced"]:
+            print(f"[{doc.meta['source_index']}] {doc.content}")
+
+    # >> Answer: The capital of France is Paris
+    # >> References:
+    # >> [2] Paris is the capital of France.
+    # >> Other sources:
+    # >> [1] Berlin is the capital of Germany.
+    # >> [3] Rome is the capital of Italy.
+    ```
+    """
+
+    def __init__(
+        self,
+        pattern: str | None = None,
+        reference_pattern: str | None = None,
+        last_message_only: bool = False,
+        *,
+        return_only_referenced_documents: bool = True,
+        expand_reference_ranges: bool = False,
+    ) -> None:
+        """
+        Creates an instance of the AnswerBuilder component.
+
+        :param pattern:
+            The regular expression pattern to extract the answer text from the Generator.
+            If not specified, the entire response is used as the answer.
+            The regular expression can have one capture group at most.
+            If present, the capture group text
+            is used as the answer. If no capture group is present, the whole match is used as the answer.
+            Examples:
+                `[^\\n]+$` finds "this is an answer" in a string "this is an argument.\\nthis is an answer".
+                `Answer: (.*)` finds "this is an answer" in a string "this is an argument. Answer: this is an answer".
+
+        :param reference_pattern:
+            The regular expression pattern used for parsing the document references.
+            If not specified, no parsing is done, and all documents are returned.
+            References need to be specified as indices of the input documents and start at [1].
+            Example: `\\[(\\d+)\\]` finds "1" in a string "this is an answer[1]".
+            If this parameter is provided, documents metadata will contain a "referenced" key with a boolean value.
+
+        :param last_message_only:
+           If False (default value), all messages are used as the answer.
+           If True, only the last message is used as the answer.
+
+        :param return_only_referenced_documents:
+            To be used in conjunction with `reference_pattern`.
+            If True (default value), only the documents that were actually referenced in `replies` are returned.
+            If False, all documents are returned.
+            If `reference_pattern` is not provided, this parameter has no effect, and all documents are returned.
+        :param expand_reference_ranges:
+            If True, reference ranges like `[6-10]` are expanded to documents 6 through 10.
+            Defaults to False for backwards compatibility.
+            When enabled with the default `reference_pattern`, a broader pattern is used automatically.
+        """
+        if pattern:
+            AnswerBuilder._check_num_groups_in_regex(pattern)
+
+        self.pattern = pattern
+        self.reference_pattern = reference_pattern
+        self.last_message_only = last_message_only
+        self.return_only_referenced_documents = return_only_referenced_documents
+        self.expand_reference_ranges = expand_reference_ranges
+
+    @component.output_types(answers=list[GeneratedAnswer])
+    def run(
+        self,
+        query: str,
+        replies: list[str] | list[ChatMessage],
+        meta: list[dict[str, Any]] | None = None,
+        documents: list[Document] | None = None,
+        pattern: str | None = None,
+        reference_pattern: str | None = None,
+        expand_reference_ranges: bool | None = None,
+    ) -> dict[str, Any]:
+        """
+        Turns the output of a Generator into `GeneratedAnswer` objects using regular expressions.
+
+        :param query:
+            The input query used as the Generator prompt.
+        :param replies:
+            The output of the Generator. Can be a list of strings or a list of `ChatMessage` objects.
+        :param meta:
+            The metadata returned by the Generator. If not specified, the generated answer will contain no metadata.
+        :param documents:
+            The documents used as the Generator inputs. If specified, they are added to
+            the `GeneratedAnswer` objects.
+            The Document copies inside the returned `GeneratedAnswer.documents` each include a "source_index" key,
+            representing the document's 1-based position in the input list. The original input documents are
+            not modified.
+            When `reference_pattern` is provided:
+            - "referenced" key is added to the Document copies inside `GeneratedAnswer.documents`, indicating if
+            the document was referenced in the output.
+            - `return_only_referenced_documents` init parameter controls if all or only referenced documents are
+            returned.
+        :param pattern:
+            The regular expression pattern to extract the answer text from the Generator.
+            If not specified, the entire response is used as the answer.
+            The regular expression can have one capture group at most.
+            If present, the capture group text
+            is used as the answer. If no capture group is present, the whole match is used as the answer.
+                Examples:
+                    `[^\\n]+$` finds "this is an answer" in a string "this is an argument.\\nthis is an answer".
+                    `Answer: (.*)` finds "this is an answer" in a string
+                    "this is an argument. Answer: this is an answer".
+        :param reference_pattern:
+            The regular expression pattern used for parsing the document references.
+            If not specified, no parsing is done, and all documents are returned.
+            References need to be specified as indices of the input documents and start at [1].
+            Example: `\\[(\\d+)\\]` finds "1" in a string "this is an answer[1]".
+        :param expand_reference_ranges:
+            If True, reference ranges like `[6-10]` are expanded to documents 6 through 10.
+            If not specified, the value from the component initialization is used.
+
+        :returns: A dictionary with the following keys:
+            - `answers`: The answers received from the output of the Generator.
+        """
+        if not meta:
+            meta = [{}] * len(replies)
+        elif len(replies) != len(meta):
+            raise ValueError(f"Number of replies ({len(replies)}), and metadata ({len(meta)}) must match.")
+
+        if pattern:
+            AnswerBuilder._check_num_groups_in_regex(pattern)
+
+        pattern = pattern or self.pattern
+        reference_pattern = reference_pattern or self.reference_pattern
+        expand_reference_ranges = (
+            self.expand_reference_ranges if expand_reference_ranges is None else expand_reference_ranges
+        )
+        reference_pattern = AnswerBuilder._resolve_reference_pattern(
+            reference_pattern=reference_pattern, expand_reference_ranges=expand_reference_ranges
+        )
+
+        replies_to_iterate = replies[-1:] if self.last_message_only and replies else replies
+        meta_to_iterate = meta[-1:] if self.last_message_only and meta else meta
+
+        all_answers = []
+        for reply, given_metadata in zip(replies_to_iterate, meta_to_iterate, strict=True):
+            # Extract content from ChatMessage objects if reply is a ChatMessages, else use the string as is
+            extracted_reply = reply.text or "" if isinstance(reply, ChatMessage) else str(reply)
+            extracted_metadata = reply.meta if isinstance(reply, ChatMessage) else {}
+
+            extracted_metadata = {**extracted_metadata, **given_metadata}
+            extracted_metadata["all_messages"] = replies
+
+            referenced_docs = []
+            if documents:
+                referenced_idxs = (
+                    AnswerBuilder._extract_reference_idxs(
+                        extracted_reply,
+                        reference_pattern,
+                        expand_ranges=expand_reference_ranges,
+                        num_documents=len(documents),
+                    )
+                    if reference_pattern
+                    else set()
+                )
+                doc_idxs = (
+                    referenced_idxs
+                    if reference_pattern and self.return_only_referenced_documents
+                    else set(range(len(documents)))
+                )
+
+                for idx in doc_idxs:
+                    try:
+                        doc = documents[idx]
+                    except IndexError:
+                        logger.warning(
+                            "Document index '{index}' referenced in Generator output is out of range. ", index=idx + 1
+                        )
+                        continue
+
+                    doc_meta: dict[str, Any] = dict(doc.meta or {})
+                    doc_meta["source_index"] = idx + 1
+                    if reference_pattern:
+                        doc_meta["referenced"] = idx in referenced_idxs
+                    referenced_docs.append(replace(doc, meta=doc_meta))
+
+            answer_string = AnswerBuilder._extract_answer_string(extracted_reply, pattern)
+            answer = GeneratedAnswer(
+                data=answer_string, query=query, documents=referenced_docs, meta=extracted_metadata
+            )
+            all_answers.append(answer)
+
+        return {"answers": all_answers}
+
+    @staticmethod
+    def _extract_answer_string(reply: str, pattern: str | None = None) -> str:
+        """
+        Extract the answer string from the generator output using the specified pattern.
+
+        If no pattern is specified, the whole string is used as the answer.
+
+        :param reply:
+            The output of the Generator. A string.
+        :param pattern:
+            The regular expression pattern to use to extract the answer text from the generator output.
+        """
+        if pattern is None:
+            return reply
+
+        if match := re.search(pattern, reply):
+            # No capture group in pattern -> use the whole match as answer
+            if not match.lastindex:
+                return match.group(0)
+            # One capture group in pattern -> use the capture group as answer
+            return match.group(1)
+        return ""
+
+    @staticmethod
+    def _resolve_reference_pattern(reference_pattern: str | None, expand_reference_ranges: bool) -> str | None:
+        if not reference_pattern or not expand_reference_ranges:
+            return reference_pattern
+        if reference_pattern == DEFAULT_REFERENCE_PATTERN:
+            return EXPANDED_REFERENCE_PATTERN
+        return reference_pattern
+
+    @staticmethod
+    def _extract_reference_idxs(
+        reply: str, reference_pattern: str, expand_ranges: bool = False, num_documents: int | None = None
+    ) -> set[int]:
+        matches = re.findall(reference_pattern, reply)
+        idxs: set[int] = set()
+        for match in matches:
+            if expand_ranges:
+                for part in match.split(","):
+                    part = part.strip()
+                    if not part:
+                        continue
+                    if "-" in part:
+                        start_str, end_str = part.split("-", 1)
+                        start, end = int(start_str), int(end_str)
+                        if start > end:
+                            continue
+                        # Clamp the range end to the number of documents to avoid materializing a huge
+                        # set from an out-of-range citation like `[1-999999999]` in the Generator output.
+                        if num_documents is not None:
+                            end = min(end, num_documents)
+                        idxs.update(range(start - 1, end))
+                    else:
+                        idxs.add(int(part) - 1)
+            else:
+                idxs.add(int(match) - 1)
+        return idxs
+
+    @staticmethod
+    def _check_num_groups_in_regex(pattern: str) -> None:
+        num_groups = re.compile(pattern).groups
+        if num_groups > 1:
+            raise ValueError(
+                f"Pattern '{pattern}' contains multiple capture groups. "
+                f"Please specify a pattern with at most one capture group."
+            )

haystack/components/builders/chat_prompt_builder.py

@@ -0,0 +1,359 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from dataclasses import replace
+from typing import Any, Literal
+
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.dataclasses.chat_message import ChatMessage, ChatRole, TextContent
+from haystack.lazy_imports import LazyImport
+from haystack.utils import Jinja2TimeExtension
+from haystack.utils.jinja2_chat_extension import ChatMessageExtension
+from haystack.utils.jinja2_extensions import _extract_template_variables_and_assignments
+
+logger = logging.getLogger(__name__)
+
+with LazyImport("Run 'pip install \"arrow>=1.3.0\"'") as arrow_import:
+    import arrow  # noqa: F401
+
+NO_TEXT_ERROR_MESSAGE = "ChatMessages from {role} role must contain text. Received ChatMessage with no text: {message}"
+
+FILTER_NOT_ALLOWED_ERROR_MESSAGE = (
+    "The templatize_part filter cannot be used with a template containing a list of"
+    "ChatMessage objects. Use a string template or remove the templatize_part filter "
+    "from the template."
+)
+
+
+@component
+class ChatPromptBuilder:
+    """
+    Renders a chat prompt from a template using Jinja2 syntax.
+
+    A template can be a list of `ChatMessage` objects, or a special string, as shown in the usage examples.
+
+    It constructs prompts using static or dynamic templates, which you can update for each pipeline run.
+
+    Template variables in the template are required by default. To make any subset of variables optional,
+    set `required_variables` to an explicit list of the variables that should remain required; any variable
+    not listed becomes optional and defaults to an empty string when missing.
+    Set `required_variables` to `None` to mark every variable as optional.
+
+    ### Usage examples
+
+    #### Static ChatMessage prompt template
+
+    ```python
+    template = [ChatMessage.from_user("Translate to {{ target_language }}. Context: {{ snippet }}; Translation:")]
+    builder = ChatPromptBuilder(template=template)
+    builder.run(target_language="spanish", snippet="I can't speak spanish.")
+    ```
+
+    #### Overriding static ChatMessage template at runtime
+
+    ```python
+    template = [ChatMessage.from_user("Translate to {{ target_language }}. Context: {{ snippet }}; Translation:")]
+    builder = ChatPromptBuilder(template=template)
+    builder.run(target_language="spanish", snippet="I can't speak spanish.")
+
+    msg = "Translate to {{ target_language }} and summarize. Context: {{ snippet }}; Summary:"
+    summary_template = [ChatMessage.from_user(msg)]
+    builder.run(target_language="spanish", snippet="I can't speak spanish.", template=summary_template)
+    ```
+
+    #### Dynamic ChatMessage prompt template
+
+    ```python
+    from haystack.components.builders import ChatPromptBuilder
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+    from haystack import Pipeline
+
+    # no parameter init, we don't use any runtime template variables
+    prompt_builder = ChatPromptBuilder()
+    llm = OpenAIChatGenerator(model="gpt-5-mini")
+
+    pipe = Pipeline()
+    pipe.add_component("prompt_builder", prompt_builder)
+    pipe.add_component("llm", llm)
+    pipe.connect("prompt_builder.prompt", "llm.messages")
+
+    location = "Berlin"
+    language = "English"
+    system_message = ChatMessage.from_system("You are an assistant giving information to tourists in {{language}}")
+    messages = [system_message, ChatMessage.from_user("Tell me about {{location}}")]
+
+    res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "language": language},
+                                        "template": messages}})
+    print(res)
+    # >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
+    # "Berlin is the capital city of Germany and one of the most vibrant
+    # and diverse cities in Europe. Here are some key things to know...Enjoy your time exploring the vibrant and dynamic
+    # capital of Germany!")], _name=None, _meta={'model': 'gpt-5-mini',
+    # 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 27, 'completion_tokens': 681, 'total_tokens':
+    # 708}})]}}
+
+    messages = [system_message, ChatMessage.from_user("What's the weather forecast for {{location}} in the next {{day_count}} days?")]
+
+    res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "day_count": "5"},
+                                        "template": messages}})
+
+    print(res)
+    # >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
+    # "Here is the weather forecast for Berlin in the next 5
+    # days:\\n\\nDay 1: Mostly cloudy with a high of 22°C (72°F) and...so it's always a good idea to check for updates
+    # closer to your visit.")], _name=None, _meta={'model': 'gpt-5-mini',
+    # 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 37, 'completion_tokens': 201,
+    # 'total_tokens': 238}})]}}
+    ```
+
+    #### String prompt template
+    ```python
+    from haystack.components.builders import ChatPromptBuilder
+    from haystack.dataclasses.image_content import ImageContent
+
+    template = \"\"\"
+    {% message role="system" %}
+    You are a helpful assistant.
+    {% endmessage %}
+
+    {% message role="user" %}
+    Hello! I am {{user_name}}. What's the difference between the following images?
+    {% for image in images %}
+    {{ image | templatize_part }}
+    {% endfor %}
+    {% endmessage %}
+    \"\"\"
+
+    images = [ImageContent.from_file_path("test/test_files/images/apple.jpg"),
+              ImageContent.from_file_path("test/test_files/images/haystack-logo.png")]
+
+    builder = ChatPromptBuilder(template=template)
+    builder.run(user_name="John", images=images)
+    ```
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        template: list[ChatMessage] | str | None = None,
+        required_variables: list[str] | Literal["*"] | None = "*",
+        variables: list[str] | None = None,
+    ) -> None:
+        """
+        Constructs a ChatPromptBuilder component.
+
+        :param template:
+            A list of `ChatMessage` objects or a string template. The component looks for Jinja2 template syntax and
+            renders the prompt with the provided variables. Provide the template in either
+            the `init` method` or the `run` method.
+        :param required_variables:
+            List variables that must be provided as input to ChatPromptBuilder.
+            Defaults to `"*"`, which marks every variable found in the prompt as required.
+            Pass an explicit list to only require a subset of the variables; any variable not listed becomes
+            optional and is replaced with an empty string in the rendered prompt when missing.
+            Set to `None` to mark every variable as optional.
+        :param variables:
+            List input variables to use in prompt templates instead of the ones inferred from the
+            `template` parameter. For example, to use more variables during prompt engineering than the ones present
+            in the default template, you can provide them here.
+        """
+        self._variables = variables
+        self._required_variables = required_variables
+        self.template = template
+
+        self._env = SandboxedEnvironment(extensions=[ChatMessageExtension])
+        if arrow_import.is_successful():
+            self._env.add_extension(Jinja2TimeExtension)
+
+        extracted_variables = []
+        if template and not variables:
+            if isinstance(template, list):
+                for message in template:
+                    if message.is_from(ChatRole.USER) or message.is_from(ChatRole.SYSTEM):
+                        # infer variables from template
+                        if message.text is None:
+                            raise ValueError(NO_TEXT_ERROR_MESSAGE.format(role=message.role.value, message=message))
+                        if message.text and "templatize_part" in message.text:
+                            raise ValueError(FILTER_NOT_ALLOWED_ERROR_MESSAGE)
+                        assigned_variables, template_variables = _extract_template_variables_and_assignments(
+                            env=self._env, template=message.text
+                        )
+                        extracted_variables += list(template_variables - assigned_variables)
+            elif isinstance(template, str):
+                assigned_variables, template_variables = _extract_template_variables_and_assignments(
+                    env=self._env, template=template
+                )
+                extracted_variables = list(template_variables - assigned_variables)
+
+        extracted_variables = extracted_variables or []
+        self.variables = variables or extracted_variables
+        self.required_variables = required_variables or []
+
+        if len(self.variables) > 0 and required_variables is None:
+            logger.warning(
+                "ChatPromptBuilder has {length} prompt variables and `required_variables` is explicitly set to "
+                "`None`. This treats all prompt variables as optional, which may lead to unintended behavior in "
+                "multi-branch pipelines. Only set `required_variables` to `None` if you intentionally want all "
+                "variables to be optional.",
+                length=len(self.variables),
+            )
+
+        # setup inputs
+        for var in self.variables:
+            if self.required_variables == "*" or var in self.required_variables:
+                component.set_input_type(self, var, Any)
+            else:
+                component.set_input_type(self, var, Any, "")
+
+    @component.output_types(prompt=list[ChatMessage])
+    def run(
+        self,
+        template: list[ChatMessage] | str | None = None,
+        template_variables: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Renders the prompt template with the provided variables.
+
+        It applies the template variables to render the final prompt. You can provide variables with pipeline kwargs.
+        To overwrite the default template, you can set the `template` parameter.
+        To overwrite pipeline kwargs, you can set the `template_variables` parameter.
+
+        :param template:
+            An optional list of `ChatMessage` objects or string template to overwrite ChatPromptBuilder's default
+            template.
+            If `None`, the default template provided at initialization is used.
+        :param template_variables:
+            An optional dictionary of template variables to overwrite the pipeline variables.
+        :param kwargs:
+            Pipeline variables used for rendering the prompt.
+
+        :returns: A dictionary with the following keys:
+            - `prompt`: The updated list of `ChatMessage` objects after rendering the templates.
+        :raises ValueError:
+            If `chat_messages` is empty or contains elements that are not instances of `ChatMessage`.
+        """
+        kwargs = kwargs or {}
+        template_variables = template_variables or {}
+        template_variables_combined = {**kwargs, **template_variables}
+
+        if template is None:
+            template = self.template
+
+        if not template:
+            raise ValueError(
+                f"The {self.__class__.__name__} requires a non-empty list of ChatMessage instances. "
+                f"Please provide a valid list of ChatMessage instances to render the prompt."
+            )
+
+        if isinstance(template, list) and not all(isinstance(message, ChatMessage) for message in template):
+            raise ValueError(
+                f"The {self.__class__.__name__} expects a list containing only ChatMessage instances. "
+                f"The provided list contains other types. Please ensure that all elements in the list "
+                f"are ChatMessage instances."
+            )
+
+        processed_messages = []
+        if isinstance(template, list):
+            for message in template:
+                if message.is_from(ChatRole.USER) or message.is_from(ChatRole.SYSTEM):
+                    self._validate_variables(set(template_variables_combined.keys()))
+                    if message.text is None:
+                        raise ValueError(NO_TEXT_ERROR_MESSAGE.format(role=message.role.value, message=message))
+                    if message.text and "templatize_part" in message.text:
+                        raise ValueError(FILTER_NOT_ALLOWED_ERROR_MESSAGE)
+                    compiled_template = self._env.from_string(message.text)
+                    rendered_text = compiled_template.render(template_variables_combined)
+                    # use dataclasses.replace to avoid in-place mutation of the original message
+                    rendered_message: ChatMessage = replace(message, _content=[TextContent(text=rendered_text)])
+                    processed_messages.append(rendered_message)
+                else:
+                    processed_messages.append(message)
+        elif isinstance(template, str):
+            self._validate_variables(set(template_variables_combined.keys()))
+            processed_messages = self._render_chat_messages_from_str_template(template, template_variables_combined)
+
+        return {"prompt": processed_messages}
+
+    def _render_chat_messages_from_str_template(
+        self, template: str, template_variables: dict[str, Any]
+    ) -> list[ChatMessage]:
+        """
+        Renders a chat message from a string template.
+
+        This must be used in conjunction with the `ChatMessageExtension` Jinja2 extension
+        and the `templatize_part` filter.
+        """
+        compiled_template = self._env.from_string(template)
+        rendered = compiled_template.render(template_variables)
+
+        messages = []
+        for line in rendered.strip().split("\n"):
+            line = line.strip()
+            if line:
+                messages.append(ChatMessage.from_dict(json.loads(line)))
+
+        return messages
+
+    def _validate_variables(self, provided_variables: set[str]) -> None:
+        """
+        Checks if all the required template variables are provided.
+
+        :param provided_variables:
+            A set of provided template variables.
+        :raises ValueError:
+            If no template is provided or if all the required template variables are not provided.
+        """
+        if self.required_variables == "*":
+            required_variables = sorted(self.variables)
+        else:
+            required_variables = self.required_variables
+        missing_variables = [var for var in required_variables if var not in provided_variables]
+        if missing_variables:
+            missing_vars_str = ", ".join(missing_variables)
+            raise ValueError(
+                f"Missing required input variables in ChatPromptBuilder: {missing_vars_str}. "
+                f"Required variables: {required_variables}. Provided variables: {provided_variables}."
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Returns a dictionary representation of the component.
+
+        :returns:
+            Serialized dictionary representation of the component.
+        """
+        template: list[dict[str, Any]] | str | None = None
+        if isinstance(self.template, list):
+            template = [m.to_dict() for m in self.template]
+        elif isinstance(self.template, str):
+            template = self.template
+
+        return default_to_dict(
+            self, template=template, variables=self._variables, required_variables=self._required_variables
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ChatPromptBuilder":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize and create the component.
+
+        :returns:
+            The deserialized component.
+        """
+        init_parameters = data["init_parameters"]
+        template = init_parameters.get("template")
+        if template:
+            if isinstance(template, list):
+                init_parameters["template"] = [ChatMessage.from_dict(d) for d in template]
+            elif isinstance(template, str):
+                init_parameters["template"] = template
+
+        return default_from_dict(cls, data)

haystack/components/builders/prompt_builder.py

@@ -0,0 +1,271 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Literal
+
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import component, default_to_dict, logging
+from haystack.utils import Jinja2TimeExtension
+from haystack.utils.jinja2_extensions import _extract_template_variables_and_assignments
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class PromptBuilder:
+    """
+
+    Renders a prompt filling in any variables so that it can send it to a Generator.
+
+    The prompt uses Jinja2 template syntax.
+    The variables in the default template are used as PromptBuilder's input and are all required by default.
+    To make any subset of variables optional, set `required_variables` to an explicit list of the variables that
+    should remain required. Optional variables are replaced with an empty string in the rendered prompt.
+    To try out different prompts, you can replace the prompt template at runtime by
+    providing a template for each pipeline run invocation.
+
+    ### Usage examples
+
+    #### On its own
+
+    This example uses PromptBuilder to render a prompt template and fill it with `target_language`
+    and `snippet`. PromptBuilder returns a prompt with the string "Translate the following context to Spanish.
+    Context: I can't speak Spanish.; Translation:".
+    ```python
+    from haystack.components.builders import PromptBuilder
+
+    template = "Translate the following context to {{ target_language }}. Context: {{ snippet }}; Translation:"
+    builder = PromptBuilder(template=template)
+    builder.run(target_language="spanish", snippet="I can't speak spanish.")
+    ```
+
+    #### In a Pipeline
+
+    This is an example of a RAG pipeline where PromptBuilder renders a custom prompt template and fills it
+    with the contents of the retrieved documents and a query. The rendered prompt is then sent to a ChatGenerator.
+    ```python
+    from haystack import Pipeline, Document
+    from haystack.utils import Secret
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.builders.prompt_builder import PromptBuilder
+
+    # in a real world use case documents could come from a retriever, web, or any other source
+    documents = [Document(content="Joe lives in Berlin"), Document(content="Joe is a software engineer")]
+    prompt_template = \"\"\"
+        Given these documents, answer the question.
+        Documents:
+        {% for doc in documents %}
+            {{ doc.content }}
+        {% endfor %}
+
+        Question: {{query}}
+        Answer:
+        \"\"\"
+    p = Pipeline()
+    p.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder")
+    p.add_component(instance=OpenAIChatGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), name="llm")
+    p.connect("prompt_builder", "llm")
+
+    question = "Where does Joe live?"
+    result = p.run({"prompt_builder": {"documents": documents, "query": question}})
+    print(result)
+    ```
+
+    #### Changing the template at runtime (prompt engineering)
+
+    You can change the prompt template of an existing pipeline, like in this example:
+    ```python
+    documents = [
+        Document(content="Joe lives in Berlin", meta={"name": "doc1"}),
+        Document(content="Joe is a software engineer", meta={"name": "doc1"}),
+    ]
+    new_template = \"\"\"
+        You are a helpful assistant.
+        Given these documents, answer the question.
+        Documents:
+        {% for doc in documents %}
+            Document {{ loop.index }}:
+            Document name: {{ doc.meta['name'] }}
+            {{ doc.content }}
+        {% endfor %}
+
+        Question: {{ query }}
+        Answer:
+        \"\"\"
+    p.run({
+        "prompt_builder": {
+            "documents": documents,
+            "query": question,
+            "template": new_template,
+        },
+    })
+    ```
+    To replace the variables in the default template when testing your prompt,
+    pass the new variables in the `variables` parameter.
+
+    #### Overwriting variables at runtime
+
+    To overwrite the values of variables, use `template_variables` during runtime:
+    ```python
+    language_template = \"\"\"
+    You are a helpful assistant.
+    Given these documents, answer the question.
+    Documents:
+    {% for doc in documents %}
+        Document {{ loop.index }}:
+        Document name: {{ doc.meta['name'] }}
+        {{ doc.content }}
+    {% endfor %}
+
+    Question: {{ query }}
+    Please provide your answer in {{ answer_language | default('English') }}
+    Answer:
+    \"\"\"
+    p.run({
+        "prompt_builder": {
+            "documents": documents,
+            "query": question,
+            "template": language_template,
+            "template_variables": {"answer_language": "German"},
+        },
+    })
+    ```
+    Note that `language_template` introduces variable `answer_language` which is not bound to any pipeline variable.
+    If not set otherwise, it will use its default value 'English'.
+    This example overwrites its value to 'German'.
+    Use `template_variables` to overwrite pipeline variables (such as documents) as well.
+
+    """
+
+    def __init__(
+        self,
+        template: str,
+        required_variables: list[str] | Literal["*"] | None = "*",
+        variables: list[str] | None = None,
+    ) -> None:
+        """
+        Constructs a PromptBuilder component.
+
+        :param template:
+            A prompt template that uses Jinja2 syntax to add variables. For example:
+            `"Summarize this document: {{ documents[0].content }}\\nSummary:"`
+            It's used to render the prompt.
+            The variables in the default template are input for PromptBuilder and are all required by default.
+        :param required_variables: List variables that must be provided as input to PromptBuilder.
+            Defaults to `"*"`, which marks every variable found in the prompt as required.
+            Pass an explicit list to only require a subset of the variables; any variable not listed becomes
+            optional and is replaced with an empty string in the rendered prompt when missing.
+            Set to `None` to mark every variable as optional.
+        :param variables:
+            List input variables to use in prompt templates instead of the ones inferred from the
+            `template` parameter. For example, to use more variables during prompt engineering than the ones present
+            in the default template, you can provide them here.
+        """
+        self._template_string = template
+        self._variables = variables
+        self._required_variables = required_variables
+        self.required_variables = required_variables or []
+        try:
+            # The Jinja2TimeExtension needs an optional dependency to be installed.
+            # If it's not available we can do without it and use the PromptBuilder as is.
+            self._env = SandboxedEnvironment(extensions=[Jinja2TimeExtension])
+        except ImportError:
+            self._env = SandboxedEnvironment()
+
+        self.template = self._env.from_string(template)
+
+        if not variables:
+            assigned_variables, template_variables = _extract_template_variables_and_assignments(
+                env=self._env, template=template
+            )
+            variables = list(template_variables - assigned_variables)
+
+        variables = variables or []
+        self.variables = variables
+
+        if len(self.variables) > 0 and required_variables is None:
+            logger.warning(
+                "PromptBuilder has {length} prompt variables and `required_variables` is explicitly set to `None`. "
+                "This treats all prompt variables as optional, which may lead to unintended behavior in "
+                "multi-branch pipelines. Only set `required_variables` to `None` if you intentionally want all "
+                "variables to be optional.",
+                length=len(self.variables),
+            )
+
+        # setup inputs
+        for var in self.variables:
+            if self.required_variables == "*" or var in self.required_variables:
+                component.set_input_type(self, var, Any)
+            else:
+                component.set_input_type(self, var, Any, "")
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Returns a dictionary representation of the component.
+
+        :returns:
+            Serialized dictionary representation of the component.
+        """
+        return default_to_dict(
+            self, template=self._template_string, variables=self._variables, required_variables=self._required_variables
+        )
+
+    @component.output_types(prompt=str)
+    def run(
+        self, template: str | None = None, template_variables: dict[str, Any] | None = None, **kwargs: Any
+    ) -> dict[str, Any]:
+        """
+        Renders the prompt template with the provided variables.
+
+        It applies the template variables to render the final prompt. You can provide variables via pipeline kwargs.
+        In order to overwrite the default template, you can set the `template` parameter.
+        In order to overwrite pipeline kwargs, you can set the `template_variables` parameter.
+
+        :param template:
+            An optional string template to overwrite PromptBuilder's default template. If None, the default template
+            provided at initialization is used.
+        :param template_variables:
+            An optional dictionary of template variables to overwrite the pipeline variables.
+        :param kwargs:
+            Pipeline variables used for rendering the prompt.
+
+        :returns: A dictionary with the following keys:
+            - `prompt`: The updated prompt text after rendering the prompt template.
+
+        :raises ValueError:
+            If any of the required template variables is not provided.
+        """
+        kwargs = kwargs or {}
+        template_variables = template_variables or {}
+        template_variables_combined = {**kwargs, **template_variables}
+        self._validate_variables(set(template_variables_combined.keys()))
+
+        compiled_template = self.template
+        if template is not None:
+            compiled_template = self._env.from_string(template)
+
+        result = compiled_template.render(template_variables_combined)
+        return {"prompt": result}
+
+    def _validate_variables(self, provided_variables: set[str]) -> None:
+        """
+        Checks if all the required template variables are provided.
+
+        :param provided_variables:
+            A set of provided template variables.
+        :raises ValueError:
+            If any of the required template variables is not provided.
+        """
+        if self.required_variables == "*":
+            required_variables = sorted(self.variables)
+        else:
+            required_variables = self.required_variables
+        missing_variables = [var for var in required_variables if var not in provided_variables]
+        if missing_variables:
+            missing_vars_str = ", ".join(missing_variables)
+            raise ValueError(
+                f"Missing required input variables in PromptBuilder: {missing_vars_str}. "
+                f"Required variables: {required_variables}. Provided variables: {provided_variables}."
+            )

haystack/components/caching/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"cache_checker": ["CacheChecker"]}
+
+if TYPE_CHECKING:
+    from .cache_checker import CacheChecker as CacheChecker
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/caching/cache_checker.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.document_stores.types import DocumentStore
+
+
+@component
+class CacheChecker:
+    """
+    Checks for the presence of documents in a Document Store based on a specified field in each document's metadata.
+
+    If matching documents are found, they are returned as "hits". If not found in the cache, the items
+    are returned as "misses".
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.document_stores.in_memory import InMemoryDocumentStore
+    from haystack.components.caching.cache_checker import CacheChecker
+
+    docstore = InMemoryDocumentStore()
+    documents = [
+        Document(content="doc1", meta={"url": "https://example.com/1"}),
+        Document(content="doc2", meta={"url": "https://example.com/2"}),
+        Document(content="doc3", meta={"url": "https://example.com/1"}),
+        Document(content="doc4", meta={"url": "https://example.com/2"}),
+    ]
+    docstore.write_documents(documents)
+    checker = CacheChecker(docstore, cache_field="url")
+    results = checker.run(items=["https://example.com/1", "https://example.com/5"])
+    assert results == {"hits": [documents[0], documents[2]], "misses": ["https://example.com/5"]}
+    ```
+    """
+
+    def __init__(self, document_store: DocumentStore, cache_field: str) -> None:
+        """
+        Creates a CacheChecker component.
+
+        :param document_store:
+            Document Store to check for the presence of specific documents.
+        :param cache_field:
+            Name of the document's metadata field
+            to check for cache hits.
+        """
+        self.document_store = document_store
+        self.cache_field = cache_field
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_store=self.document_store, cache_field=self.cache_field)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "CacheChecker":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    @component.output_types(hits=list[Document], misses=list)
+    def run(self, items: list[Any]) -> dict[str, Any]:
+        """
+        Checks if any document associated with the specified cache field is already present in the store.
+
+        :param items:
+            Values to be checked against the cache field.
+        :return:
+            A dictionary with two keys:
+            - `hits` - Documents that matched with at least one of the items.
+            - `misses` - Items that were not present in any documents.
+        """
+        found_documents = []
+        misses = []
+
+        for item in items:
+            filters = {"field": self.cache_field, "operator": "==", "value": item}
+            found = self.document_store.filter_documents(filters=filters)
+            if found:
+                found_documents.extend(found)
+            else:
+                misses.append(item)
+        return {"hits": found_documents, "misses": misses}
+
+    @component.output_types(hits=list[Document], misses=list)
+    async def run_async(self, items: list[Any]) -> dict[str, Any]:
+        """
+        Asynchronously checks if any document associated with the specified cache field is already present in the store.
+
+        :param items:
+            Values to be checked against the cache field.
+        :return:
+            A dictionary with two keys:
+            - `hits` - Documents that matched with at least one of the items.
+            - `misses` - Items that were not present in any documents.
+        """
+        found_documents = []
+        misses = []
+
+        if not hasattr(self.document_store, "filter_documents_async"):
+            raise TypeError(f"Document store {type(self.document_store).__name__} does not provide async support.")
+
+        for item in items:
+            filters = {"field": self.cache_field, "operator": "==", "value": item}
+            found = await self.document_store.filter_documents_async(filters=filters)
+            if found:
+                found_documents.extend(found)
+            else:
+                misses.append(item)
+        return {"hits": found_documents, "misses": misses}

haystack/components/converters/__init__.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "csv": ["CSVToDocument"],
+    "docx": ["DOCXToDocument"],
+    "file_to_file_content": ["FileToFileContent"],
+    "html": ["HTMLToDocument"],
+    "json": ["JSONConverter"],
+    "markdown": ["MarkdownToDocument"],
+    "msg": ["MSGToDocument"],
+    "multi_file_converter": ["MultiFileConverter"],
+    "output_adapter": ["OutputAdapter"],
+    "pdfminer": ["PDFMinerToDocument"],
+    "pptx": ["PPTXToDocument"],
+    "pypdf": ["PyPDFToDocument"],
+    "txt": ["TextFileToDocument"],
+    "xlsx": ["XLSXToDocument"],
+}
+
+if TYPE_CHECKING:
+    from .csv import CSVToDocument as CSVToDocument
+    from .docx import DOCXToDocument as DOCXToDocument
+    from .file_to_file_content import FileToFileContent as FileToFileContent
+    from .html import HTMLToDocument as HTMLToDocument
+    from .json import JSONConverter as JSONConverter
+    from .markdown import MarkdownToDocument as MarkdownToDocument
+    from .msg import MSGToDocument as MSGToDocument
+    from .multi_file_converter import MultiFileConverter as MultiFileConverter
+    from .output_adapter import OutputAdapter as OutputAdapter
+    from .pdfminer import PDFMinerToDocument as PDFMinerToDocument
+    from .pptx import PPTXToDocument as PPTXToDocument
+    from .pypdf import PyPDFToDocument as PyPDFToDocument
+    from .txt import TextFileToDocument as TextFileToDocument
+    from .xlsx import XLSXToDocument as XLSXToDocument
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/converters/csv.py

@@ -0,0 +1,238 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import io
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+
+logger = logging.getLogger(__name__)
+
+_ROW_MODE_SIZE_WARN_BYTES = 5 * 1024 * 1024  # ~5MB; warn when parsing rows might be memory-heavy
+
+
+@component
+class CSVToDocument:
+    """
+    Converts CSV files to Documents.
+
+    By default, it uses UTF-8 encoding when converting files but
+    you can also set a custom encoding.
+    It can attach metadata to the resulting documents.
+
+    ### Usage example
+    ```python
+    from haystack.components.converters.csv import CSVToDocument
+    from datetime import datetime
+
+    converter = CSVToDocument()
+    results = converter.run(
+        sources=["test/test_files/csv/sample_1.csv"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >>  'col1,col2\\nrow1,row1\\nrow2,row2\\n'
+    ```
+    """
+
+    def __init__(
+        self,
+        encoding: str = "utf-8",
+        store_full_path: bool = False,
+        *,
+        conversion_mode: Literal["file", "row"] = "file",
+        delimiter: str = ",",
+        quotechar: str = '"',
+    ) -> None:
+        """
+        Creates a CSVToDocument component.
+
+        :param encoding:
+            The encoding of the csv files to convert.
+            If the encoding is specified in the metadata of a source ByteStream,
+            it overrides this value.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        :param conversion_mode:
+            - "file" (default): one Document per CSV file whose content is the raw CSV text.
+            - "row": convert each CSV row to its own Document (requires `content_column` in `run()`).
+        :param delimiter:
+            CSV delimiter used when parsing in row mode (passed to ``csv.DictReader``).
+        :param quotechar:
+            CSV quote character used when parsing in row mode (passed to ``csv.DictReader``).
+        """
+        self.encoding = encoding
+        self.store_full_path = store_full_path
+        self.conversion_mode = conversion_mode
+        self.delimiter = delimiter
+        self.quotechar = quotechar
+
+        # Basic validation
+        if len(self.delimiter) != 1:
+            raise ValueError("CSVToDocument: delimiter must be a single character.")
+        if len(self.quotechar) != 1:
+            raise ValueError("CSVToDocument: quotechar must be a single character.")
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self,
+        sources: list[str | Path | ByteStream],
+        *,
+        content_column: str | None = None,
+        meta: dict[str, Any] | list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Converts CSV files to a Document (file mode) or to one Document per row (row mode).
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param content_column:
+            **Required when** ``conversion_mode="row"``.
+            The column name whose values become ``Document.content`` for each row.
+            The column must exist in the CSV header.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output documents.
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created documents
+        """
+        documents: list[Document] = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            try:
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                raw = io.BytesIO(bytestream.data).getvalue()
+                data = raw.decode(encoding=encoding)
+            except Exception as e:
+                logger.warning(
+                    "Could not convert file {source}. Skipping it. Error message: {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and "file_path" in bytestream.meta:
+                file_path = bytestream.meta.get("file_path")
+                if file_path:  # Ensure the value is not None for mypy
+                    merged_metadata["file_path"] = os.path.basename(file_path)
+
+            # Mode: file (backward-compatible default) -> one Document per file
+            if self.conversion_mode == "file":
+                documents.append(Document(content=data, meta=merged_metadata))
+                continue
+
+            # --- ROW MODE (strict) ---
+            # Require content_column in run(); no fallback
+            if not content_column:
+                raise ValueError(
+                    "CSVToDocument(row): 'content_column' is required in run() when conversion_mode='row'."
+                )
+
+            # Warn for large CSVs in row mode (memory consideration)
+            try:
+                size_bytes = len(raw)
+                if size_bytes > _ROW_MODE_SIZE_WARN_BYTES:
+                    logger.warning(
+                        "CSVToDocument(row): parsing a large CSV (~{mb:.1f} MB). "
+                        "Consider chunking/streaming if you hit memory issues.",
+                        mb=size_bytes / (1024 * 1024),
+                    )
+            except Exception:
+                pass
+
+            # Create DictReader; if this fails, raise (no fallback)
+            try:
+                # ``restkey`` ensures surplus fields on ragged rows (rows with more values than the
+                # header, e.g. an unquoted comma inside a value) land under an explicit string key
+                # instead of the default ``None`` key, which would break ``Document`` id generation.
+                reader = csv.DictReader(
+                    io.StringIO(data), delimiter=self.delimiter, quotechar=self.quotechar, restkey="extra_columns"
+                )
+            except Exception as e:
+                raise RuntimeError(f"CSVToDocument(row): could not parse CSV rows for {source}: {e}") from e
+
+            # Validate header contains content_column; strict error if missing
+            header = reader.fieldnames or []
+            if content_column not in header:
+                raise ValueError(
+                    f"CSVToDocument(row): content_column='{content_column}' not found in header "
+                    f"for {source}. Available columns: {header}"
+                )
+
+            # Build documents; if a row processing fails, raise immediately (no skip)
+            for i, row in enumerate(reader):
+                try:
+                    doc = self._build_document_from_row(
+                        row=row, base_meta=merged_metadata, row_index=i, content_column=content_column
+                    )
+                except Exception as e:
+                    raise RuntimeError(f"CSVToDocument(row): failed to process row {i} for {source}: {e}") from e
+                documents.append(doc)
+
+        return {"documents": documents}
+
+    # ----- helpers -----
+    def _safe_value(self, value: Any) -> str:
+        """Normalize CSV cell values: None -> '', everything -> str."""
+        return "" if value is None else str(value)
+
+    def _build_document_from_row(
+        self, row: dict[str, Any], base_meta: dict[str, Any], row_index: int, content_column: str
+    ) -> Document:
+        """
+        Build a ``Document`` from one parsed CSV row.
+
+        :param row: Mapping of column name to cell value for the current row
+            (as produced by ``csv.DictReader``).
+        :param base_meta: File-level and user-provided metadata to start from
+            (for example: ``file_path``, ``encoding``).
+        :param row_index: Zero-based row index in the CSV; stored as
+            ``row_number`` in the output document's metadata.
+        :param content_column: Column name to use for ``Document.content``.
+        :returns: A ``Document`` with chosen content and merged metadata.
+            Remaining row columns are added to ``meta`` with collision-safe
+            keys (prefixed with ``csv_`` if needed).
+        """
+        row_meta = dict(base_meta)
+
+        # content (strict: content_column must exist; validated by caller)
+        content = self._safe_value(row.get(content_column))
+
+        # merge remaining columns into meta with collision handling
+        for k, v in row.items():
+            if k == content_column:
+                continue
+            key_to_use = k
+            if key_to_use in row_meta:
+                # Avoid clobbering existing meta like file_path/encoding; prefix and de-dupe
+                base_key = f"csv_{key_to_use}"
+                key_to_use = base_key
+                suffix = 1
+                while key_to_use in row_meta:
+                    key_to_use = f"{base_key}_{suffix}"
+                    suffix += 1
+            row_meta[key_to_use] = self._safe_value(v)
+
+        row_meta["row_number"] = row_index
+        return Document(content=content, meta=row_meta)

haystack/components/converters/docx.py

@@ -0,0 +1,410 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import io
+import os
+from dataclasses import asdict, dataclass
+from enum import Enum
+from io import StringIO
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+logger = logging.getLogger(__name__)
+
+with LazyImport("Run 'pip install python-docx'") as docx_import:
+    import docx
+    from docx.document import Document as DocxDocument
+    from docx.table import Table
+    from docx.text.hyperlink import Hyperlink
+    from docx.text.paragraph import Paragraph
+    from docx.text.run import Run
+    from lxml.etree import _Comment
+
+
+@dataclass
+class DOCXMetadata:
+    """
+    Describes the metadata of Docx file.
+
+    :param author: The author
+    :param category: The category
+    :param comments: The comments
+    :param content_status: The content status
+    :param created: The creation date (ISO formatted string)
+    :param identifier: The identifier
+    :param keywords: Available keywords
+    :param language: The language of the document
+    :param last_modified_by: User who last modified the document
+    :param last_printed: The last printed date (ISO formatted string)
+    :param modified: The last modification date (ISO formatted string)
+    :param revision: The revision number
+    :param subject: The subject
+    :param title: The title
+    :param version: The version
+    """
+
+    author: str
+    category: str
+    comments: str
+    content_status: str
+    created: str | None
+    identifier: str
+    keywords: str
+    language: str
+    last_modified_by: str
+    last_printed: str | None
+    modified: str | None
+    revision: int
+    subject: str
+    title: str
+    version: str
+
+
+class DOCXTableFormat(Enum):
+    """
+    Supported formats for storing DOCX tabular data in a Document.
+    """
+
+    MARKDOWN = "markdown"
+    CSV = "csv"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "DOCXTableFormat":
+        """
+        Convert a string to a DOCXTableFormat enum.
+        """
+        enum_map = {e.value: e for e in DOCXTableFormat}
+        table_format = enum_map.get(string.lower())
+        if table_format is None:
+            msg = f"Unknown table format '{string}'. Supported formats are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return table_format
+
+
+class DOCXLinkFormat(Enum):
+    """
+    Supported formats for storing DOCX link information in a Document.
+    """
+
+    MARKDOWN = "markdown"
+    PLAIN = "plain"
+    NONE = "none"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "DOCXLinkFormat":
+        """
+        Convert a string to a DOCXLinkFormat enum.
+        """
+        enum_map = {e.value: e for e in DOCXLinkFormat}
+        link_format = enum_map.get(string.lower())
+        if link_format is None:
+            msg = f"Unknown link format '{string}'. Supported formats are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return link_format
+
+
+@component
+class DOCXToDocument:
+    """
+    Converts DOCX files to Documents.
+
+    Uses `python-docx` library to convert the DOCX file to a document.
+    This component does not preserve page breaks in the original document.
+
+    Usage example:
+
+    ```python
+    from haystack.components.converters.docx import DOCXToDocument, DOCXTableFormat, DOCXLinkFormat
+    from datetime import datetime
+
+    converter = DOCXToDocument(table_format=DOCXTableFormat.CSV, link_format=DOCXLinkFormat.MARKDOWN)
+    results = converter.run(
+        sources=["test/test_files/docx/sample_docx.docx"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> 'This is a text from the DOCX file.'
+    ```
+    """
+
+    def __init__(
+        self,
+        table_format: str | DOCXTableFormat = DOCXTableFormat.CSV,
+        link_format: str | DOCXLinkFormat = DOCXLinkFormat.NONE,
+        store_full_path: bool = False,
+    ) -> None:
+        """
+        Create a DOCXToDocument component.
+
+        :param table_format: The format for table output. Can be either DOCXTableFormat.MARKDOWN,
+            DOCXTableFormat.CSV, "markdown", or "csv".
+        :param link_format: The format for link output. Can be either:
+            DOCXLinkFormat.MARKDOWN or "markdown" to get `[text](address)`,
+            DOCXLinkFormat.PLAIN or "plain" to get text (address),
+            DOCXLinkFormat.NONE or "none" to get text without links.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        docx_import.check()
+        self.table_format = DOCXTableFormat.from_str(table_format) if isinstance(table_format, str) else table_format
+        self.link_format = DOCXLinkFormat.from_str(link_format) if isinstance(link_format, str) else link_format
+        self.store_full_path = store_full_path
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            table_format=str(self.table_format),
+            link_format=str(self.link_format),
+            store_full_path=self.store_full_path,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "DOCXToDocument":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        if "table_format" in data["init_parameters"]:
+            data["init_parameters"]["table_format"] = DOCXTableFormat.from_str(data["init_parameters"]["table_format"])
+        if "link_format" in data["init_parameters"]:
+            data["init_parameters"]["link_format"] = DOCXLinkFormat.from_str(data["init_parameters"]["link_format"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, Any]:
+        """
+        Converts DOCX files to Documents.
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created Documents
+        """
+        documents = []
+        meta_list = normalize_metadata(meta=meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                docx_document = docx.Document(io.BytesIO(bytestream.data))
+                elements = self._extract_elements(docx_document)
+                text = "\n".join(elements)
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to a DOCX Document, skipping. Error: {error}",
+                    source=source,
+                    error=e,
+                )
+                continue
+
+            docx_metadata = asdict(self._get_docx_metadata(document=docx_document))
+            merged_metadata = {**bytestream.meta, **metadata, "docx": docx_metadata}
+
+            if not self.store_full_path and "file_path" in bytestream.meta:
+                file_path = bytestream.meta.get("file_path")
+                if file_path:  # Ensure the value is not None for mypy
+                    merged_metadata["file_path"] = os.path.basename(file_path)
+
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}
+
+    def _extract_elements(self, document: "DocxDocument") -> list[str]:
+        """
+        Extracts elements from a DOCX file.
+
+        :param document: The DOCX Document object.
+        :returns: List of strings (paragraph texts and table representations) with page breaks added as '\f' characters.
+        """
+        elements = []
+        for element in document.element.body:
+            if isinstance(element, _Comment):
+                continue
+            if element.tag.endswith("p"):
+                paragraph = Paragraph(element, document)
+                if paragraph.contains_page_break:
+                    para_text = self._process_paragraph_with_page_breaks(paragraph)
+                else:
+                    para_text = self._process_links_in_paragraph(paragraph)
+                elements.append(para_text)
+            elif element.tag.endswith("tbl"):
+                table = docx.table.Table(element, document)
+                table_str = (
+                    self._table_to_markdown(table)
+                    if self.table_format == DOCXTableFormat.MARKDOWN
+                    else self._table_to_csv(table)
+                )
+                elements.append(table_str)
+
+        return elements
+
+    def _process_paragraph_with_page_breaks(self, paragraph: "Paragraph") -> str:
+        """
+        Processes a paragraph with page breaks.
+
+        :param paragraph: The DOCX paragraph to process.
+        :returns: A string with page breaks added as '\f' characters.
+        """
+        para_text = ""
+        # Usually, just 1 page break exists, but could be more if paragraph is really long, so we loop over them
+        for pb_index, page_break in enumerate(paragraph.rendered_page_breaks):
+            # Can only extract text from first paragraph page break, unfortunately
+            if pb_index == 0:
+                if page_break.preceding_paragraph_fragment:
+                    para_text += self._process_links_in_paragraph(page_break.preceding_paragraph_fragment)
+                para_text += "\f"
+                if page_break.following_paragraph_fragment:
+                    # following_paragraph_fragment contains all text for remainder of paragraph.
+                    # However, if the remainder of the paragraph spans multiple page breaks, it won't include
+                    # those later page breaks so we have to add them at end of text in the `else` block below.
+                    # This is not ideal, but this case should be very rare and this is likely good enough.
+                    para_text += self._process_links_in_paragraph(page_break.following_paragraph_fragment)
+            else:
+                para_text += "\f"
+        return para_text
+
+    def _process_links_in_paragraph(self, paragraph: "Paragraph") -> str:
+        """
+        Processes links in a paragraph and formats them according to the specified link format.
+
+        :param paragraph: The DOCX paragraph to process.
+        :returns: A string with links formatted according to the specified format.
+        """
+        if self.link_format == DOCXLinkFormat.NONE:
+            return paragraph.text
+        text = ""
+        # Iterate over all hyperlinks and other content in the paragraph
+        # https://python-docx.readthedocs.io/en/latest/api/text.html#docx.text.paragraph.Paragraph.iter_inner_content
+        for content in paragraph.iter_inner_content():
+            if isinstance(content, Run):
+                text += content.text
+            elif isinstance(content, Hyperlink):
+                if self.link_format == DOCXLinkFormat.MARKDOWN:
+                    formatted_link = f"[{content.text}]({content.address})"
+                else:  # PLAIN format
+                    formatted_link = f"{content.text} ({content.address})"
+                text += formatted_link
+
+        return text
+
+    def _table_to_markdown(self, table: "Table") -> str:
+        """
+        Converts a DOCX table to a Markdown string.
+
+        :param table: The DOCX table to convert.
+        :returns: A Markdown string representation of the table.
+        """
+        markdown: list[str] = []
+        max_col_widths: list[int] = []
+
+        # Calculate max width for each column
+        for row in table.rows:
+            for i, cell in enumerate(row.cells):
+                cell_text = cell.text.strip()
+                if i >= len(max_col_widths):
+                    max_col_widths.append(len(cell_text))
+                else:
+                    max_col_widths[i] = max(max_col_widths[i], len(cell_text))
+
+        # Process rows
+        for i, row in enumerate(table.rows):
+            md_row = [cell.text.strip().ljust(max_col_widths[j]) for j, cell in enumerate(row.cells)]
+            markdown.append("| " + " | ".join(md_row) + " |")
+
+            # Add separator after header row
+            if i == 0:
+                separator = ["-" * max_col_widths[j] for j in range(len(row.cells))]
+                markdown.append("| " + " | ".join(separator) + " |")
+
+        return "\n".join(markdown)
+
+    def _table_to_csv(self, table: "Table") -> str:
+        """
+        Converts a DOCX table to a CSV string.
+
+        :param table: The DOCX table to convert.
+        :returns: A CSV string representation of the table.
+        """
+        csv_output = StringIO()
+        csv_writer = csv.writer(csv_output, quoting=csv.QUOTE_MINIMAL)
+
+        # Process rows
+        for row in table.rows:
+            csv_row = [cell.text.strip() for cell in row.cells]
+            csv_writer.writerow(csv_row)
+
+        # Get the CSV as a string and strip any trailing newlines
+        csv_string = csv_output.getvalue().strip()
+        csv_output.close()
+
+        return csv_string
+
+    def _get_docx_metadata(self, document: "DocxDocument") -> DOCXMetadata:
+        """
+        Get all relevant data from the 'core_properties' attribute from a DOCX Document.
+
+        :param document:
+            The DOCX Document you want to extract metadata from
+
+        :returns:
+            A `DOCXMetadata` dataclass all the relevant fields from the 'core_properties'
+        """
+        return DOCXMetadata(
+            author=document.core_properties.author,
+            category=document.core_properties.category,
+            comments=document.core_properties.comments,
+            content_status=document.core_properties.content_status,
+            created=(document.core_properties.created.isoformat() if document.core_properties.created else None),
+            identifier=document.core_properties.identifier,
+            keywords=document.core_properties.keywords,
+            language=document.core_properties.language,
+            last_modified_by=document.core_properties.last_modified_by,
+            last_printed=(
+                document.core_properties.last_printed.isoformat() if document.core_properties.last_printed else None
+            ),
+            modified=(document.core_properties.modified.isoformat() if document.core_properties.modified else None),
+            revision=document.core_properties.revision,
+            subject=document.core_properties.subject,
+            title=document.core_properties.title,
+            version=document.core_properties.version,
+        )

haystack/components/converters/file_to_file_content.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import base64
+from pathlib import Path
+from typing import Any
+
+from haystack import component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream, FileContent
+
+logger = logging.getLogger(__name__)
+
+
+_EMPTY_BYTE_STRING = b""
+
+
+@component
+class FileToFileContent:
+    """
+    Converts files to FileContent objects to be included in ChatMessage objects.
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.converters import FileToFileContent
+
+    converter = FileToFileContent()
+    sources = ["test/test_files/pdf/react_paper.pdf", "test/test_files/images/haystack-logo.png"]
+    file_contents = converter.run(sources=sources)["file_contents"]
+
+    print(file_contents)
+    # >> [FileContent(base64_data='...', mime_type='application/pdf', filename='react_paper.pdf', extra={}),
+    # >>  FileContent(base64_data='...', mime_type='image/png', filename='haystack-logo.png', extra={})
+    # >>]
+    ```
+    """
+
+    @component.output_types(file_contents=list[FileContent])
+    def run(
+        self, sources: list[str | Path | ByteStream], *, extra: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[FileContent]]:
+        """
+        Converts files to FileContent objects.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param extra:
+            Optional extra information to attach to the FileContent objects. Can be used to store provider-specific
+            information.
+            To avoid serialization issues, values should be JSON serializable.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the extra of all produced FileContent objects.
+            If it's a list, its length must match the number of sources as they're zipped together.
+
+        :returns:
+            A dictionary with the following keys:
+            - `file_contents`: A list of FileContent objects.
+        """
+        if not sources:
+            return {"file_contents": []}
+
+        file_contents = []
+
+        extra_list = normalize_metadata(extra, sources_count=len(sources))
+
+        for source, extra_dict in zip(sources, extra_list, strict=True):
+            if isinstance(source, str):
+                source = Path(source)
+
+            filename = source.name if isinstance(source, Path) else None
+
+            try:
+                bytestream = get_bytestream_from_source(source, guess_mime_type=True)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            if bytestream.data == _EMPTY_BYTE_STRING:
+                logger.warning("File {source} is empty. Skipping it.", source=source)
+                continue
+
+            base64_data = base64.b64encode(bytestream.data).decode("utf-8")
+            # ``normalize_metadata`` returns the same dict object for every source when ``extra`` is a
+            # single dict (or ``None``), so give each FileContent its own copy. Otherwise mutating one
+            # file's ``extra`` downstream would leak into all the others. The other converters avoid this
+            # implicitly by merging ``extra`` into a fresh ``{**bytestream.meta, ...}`` dict.
+            file_content = FileContent(
+                base64_data=base64_data, mime_type=bytestream.mime_type, filename=filename, extra=dict(extra_dict)
+            )
+            file_contents.append(file_content)
+
+        return {"file_contents": file_contents}

haystack/components/converters/html.py

@@ -0,0 +1,147 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+logger = logging.getLogger(__name__)
+
+with LazyImport("Run 'pip install trafilatura'") as trafilatura_import:
+    from trafilatura import extract
+
+
+@component
+class HTMLToDocument:
+    """
+    Converts an HTML file to a Document.
+
+    Usage example:
+    ```python
+    from haystack.components.converters import HTMLToDocument
+
+    converter = HTMLToDocument()
+    results = converter.run(sources=["test/test_files/html/paul_graham_superlinear.html"])
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> 'This is a text from the HTML file.'
+    ```
+    """
+
+    def __init__(
+        self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False, encoding: str = "utf-8"
+    ) -> None:
+        """
+        Create an HTMLToDocument component.
+
+        :param extraction_kwargs: A dictionary containing keyword arguments to customize the extraction process. These
+            are passed to the underlying Trafilatura `extract` function. For the full list of available arguments, see
+            the [Trafilatura documentation](https://trafilatura.readthedocs.io/en/latest/corefunctions.html#extract).
+        :param store_full_path:
+        If True, the full path of the file is stored in the metadata of the document.
+        If False, only the file name is stored.
+        :param encoding:
+            The default encoding to use when converting HTML files. If the encoding is specified in the metadata of a
+            source ByteStream, it overrides this value.
+        """
+        trafilatura_import.check()
+
+        self.extraction_kwargs = extraction_kwargs or {}
+        self.store_full_path = store_full_path
+        self.encoding = encoding
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path, encoding=self.encoding
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "HTMLToDocument":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self,
+        sources: list[str | Path | ByteStream],
+        meta: dict[str, Any] | list[dict[str, Any]] | None = None,
+        extraction_kwargs: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Converts a list of HTML files to Documents.
+
+        :param sources:
+            List of HTML file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+        :param extraction_kwargs:
+            Additional keyword arguments to customize the extraction process.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created Documents
+        """
+
+        merged_extraction_kwargs = {**self.extraction_kwargs, **(extraction_kwargs or {})}
+
+        documents = []
+        meta_list = normalize_metadata(meta=meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source=source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            if not bytestream.data:
+                logger.warning("Skipping {source} because it is empty.", source=source)
+                continue
+
+            try:
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                text = extract(bytestream.data.decode(encoding), **merged_extraction_kwargs)
+            except Exception as conversion_e:
+                logger.warning(
+                    "Failed to extract text from {source}. Skipping it. Error: {error}",
+                    source=source,
+                    error=conversion_e,
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and "file_path" in bytestream.meta:
+                file_path = bytestream.meta.get("file_path")
+                if file_path:  # Ensure the value is not None for mypy
+                    merged_metadata["file_path"] = os.path.basename(file_path)
+
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/image/__init__.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "document_to_image": ["DocumentToImageContent"],
+    "file_to_document": ["ImageFileToDocument"],
+    "file_to_image": ["ImageFileToImageContent"],
+    "pdf_to_image": ["PDFToImageContent"],
+}
+
+if TYPE_CHECKING:
+    from .document_to_image import DocumentToImageContent as DocumentToImageContent
+    from .file_to_document import ImageFileToDocument as ImageFileToDocument
+    from .file_to_image import ImageFileToImageContent as ImageFileToImageContent
+    from .pdf_to_image import PDFToImageContent as PDFToImageContent
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/converters/image/document_to_image.py

@@ -0,0 +1,175 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Literal
+
+from haystack import Document, component, logging
+from haystack.components.converters.image.image_utils import (
+    _batch_convert_pdf_pages_to_images,
+    _encode_image_to_base64,
+    _extract_image_sources_info,
+    _PDFPageInfo,
+    pillow_import,
+    pypdfium2_import,
+)
+from haystack.dataclasses import ByteStream
+from haystack.dataclasses.image_content import ImageContent
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class DocumentToImageContent:
+    """
+    Converts documents sourced from PDF and image files into ImageContents.
+
+    This component processes a list of documents and extracts visual content from supported file formats, converting
+    them into ImageContents that can be used for multimodal AI tasks. It handles both direct image files and PDF
+    documents by extracting specific pages as images.
+
+    Documents are expected to have metadata containing:
+    - The `file_path_meta_field` key with a valid file path that exists when combined with `root_path`
+    - A supported image format (MIME type must be one of the supported image types)
+    - For PDF files, a `page_number` key specifying which page to extract
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.converters.image.document_to_image import DocumentToImageContent
+
+    converter = DocumentToImageContent(
+        file_path_meta_field="file_path",
+        root_path="test/test_files",
+        detail="high",
+        size=(800, 600)
+    )
+
+    documents = [
+        Document(content="Optional description of apple.jpg", meta={"file_path": "images/apple.jpg"}),
+        Document(
+            content="Optional description of sample_pdf_1.pdf",
+            meta={"file_path": "pdf/sample_pdf_1.pdf", "page_number": 1}
+        )
+    ]
+
+    result = converter.run(documents)
+    image_contents = result["image_contents"]
+    # [ImageContent(
+    #    base64_image='/9j/4A...', mime_type='image/jpeg', detail='high', meta={'file_path': 'images/apple.jpg'}
+    #  ),
+    #  ImageContent(
+    #    base64_image='/9j/4A...', mime_type='image/jpeg', detail='high',
+    #    meta={'file_path': 'pdf/sample_pdf_1.pdf', 'page_number': 1})
+    #  )]
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        file_path_meta_field: str = "file_path",
+        root_path: str | None = None,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+    ) -> None:
+        """
+        Initialize the DocumentToImageContent component.
+
+        :param file_path_meta_field: The metadata field in the Document that contains the file path to the image or PDF.
+        :param root_path: The root directory path where document files are located. If provided, file paths in
+            document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
+        :param detail: Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+        :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+        """
+        pillow_import.check()
+        pypdfium2_import.check()
+
+        self.file_path_meta_field = file_path_meta_field
+        self.root_path = root_path or ""
+        self.detail = detail
+        self.size = size
+
+    @component.output_types(image_contents=list[ImageContent | None])
+    def run(self, documents: list[Document]) -> dict[str, list[ImageContent | None]]:
+        """
+        Convert documents with image or PDF sources into ImageContent objects.
+
+        This method processes the input documents, extracting images from supported file formats and converting them
+        into ImageContent objects.
+
+        :param documents: A list of documents to process. Each document should have metadata containing at minimum
+            a 'file_path_meta_field' key. PDF documents additionally require a 'page_number' key to specify which
+            page to convert.
+
+        :returns:
+            Dictionary containing one key:
+            - "image_contents": ImageContents created from the processed documents. These contain base64-encoded image
+                data and metadata. The order corresponds to order of input documents.
+        :raises ValueError:
+            If any document is missing the required metadata keys, has an invalid file path, or has an unsupported
+            MIME type. The error message will specify which document and what information is missing or incorrect.
+        """
+        if not documents:
+            return {"image_contents": []}
+
+        images_source_info = _extract_image_sources_info(
+            documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
+        )
+
+        image_contents: list[ImageContent | None] = [None] * len(documents)
+
+        pdf_page_infos: list[_PDFPageInfo] = []
+
+        for doc_idx, image_source_info in enumerate(images_source_info):
+            mime_type = image_source_info["mime_type"]
+            path = image_source_info["path"]
+            if mime_type == "application/pdf":
+                # Store PDF documents for later processing
+                page_number = image_source_info.get("page_number")
+                assert page_number is not None  # checked in _extract_image_sources_info but mypy doesn't know that
+                pdf_page_info: _PDFPageInfo = {"doc_idx": doc_idx, "path": path, "page_number": page_number}
+                pdf_page_infos.append(pdf_page_info)
+            else:
+                # Process images directly
+                bytestream = ByteStream.from_file_path(filepath=path, mime_type=mime_type)
+                _, base64_image = _encode_image_to_base64(bytestream=bytestream, size=self.size)
+                image_contents[doc_idx] = ImageContent(
+                    base64_image=base64_image,
+                    mime_type=mime_type,
+                    detail=self.detail,
+                    meta={"file_path": documents[doc_idx].meta[self.file_path_meta_field]},
+                )
+
+        # efficiently convert PDF pages to images: each PDF is opened and processed only once
+        pdf_page_infos_by_doc_idx: dict[int, _PDFPageInfo] = {
+            pdf_page_info["doc_idx"]: pdf_page_info for pdf_page_info in pdf_page_infos
+        }
+        pdf_images_by_doc_idx = _batch_convert_pdf_pages_to_images(
+            pdf_page_infos=pdf_page_infos, size=self.size, return_base64=True
+        )
+        for doc_idx, base64_pdf_image in pdf_images_by_doc_idx.items():
+            meta = {
+                "file_path": documents[doc_idx].meta[self.file_path_meta_field],
+                "page_number": pdf_page_infos_by_doc_idx[doc_idx]["page_number"],
+            }
+            # we know that base64_pdf_image is a string because we set return_base64=True but mypy doesn't know that
+            assert isinstance(base64_pdf_image, str)
+            image_contents[doc_idx] = ImageContent(
+                base64_image=base64_pdf_image, mime_type="image/jpeg", detail=self.detail, meta=meta
+            )
+
+        none_image_contents_doc_ids = [
+            documents[doc_idx].id for doc_idx, image_content in enumerate(image_contents) if image_content is None
+        ]
+        if none_image_contents_doc_ids:
+            logger.warning(
+                "Conversion failed for some documents. Their output will be None. Document IDs: {document_ids}",
+                document_ids=none_image_contents_doc_ids,
+            )
+
+        return {"image_contents": image_contents}

haystack/components/converters/image/file_to_document.py

@@ -0,0 +1,98 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class ImageFileToDocument:
+    """
+    Converts image file references into empty Document objects with associated metadata.
+
+    This component is useful in pipelines where image file paths need to be wrapped in `Document` objects to be
+    processed by downstream components such as the `LLMDocumentContentExtractor` or the
+    `SentenceTransformersDocumentImageEmbedder` (available in the `sentence-transformers-haystack` integration).
+
+    It does **not** extract any content from the image files, instead it creates `Document` objects with `None` as
+    their content and attaches metadata such as file path and any user-provided values.
+
+    ### Usage example
+    ```python
+    from haystack.components.converters.image import ImageFileToDocument
+
+    converter = ImageFileToDocument()
+
+    sources = ["image.jpg", "another_image.png"]
+
+    result = converter.run(sources=sources)
+    documents = result["documents"]
+
+    print(documents)
+
+    # [Document(id=..., meta: {'file_path': 'image.jpg'}),
+    # Document(id=..., meta: {'file_path': 'another_image.png'})]
+    ```
+    """
+
+    def __init__(self, *, store_full_path: bool = False) -> None:
+        """
+        Initialize the ImageFileToDocument component.
+
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        self.store_full_path = store_full_path
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, *, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Convert image files into empty Document objects with metadata.
+
+        This method accepts image file references (as file paths or ByteStreams) and creates `Document` objects
+        without content. These documents are enriched with metadata derived from the input source and optional
+        user-provided metadata.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, its length must match the number of sources, as they are zipped together.
+            For ByteStream objects, their `meta` is added to the output documents.
+
+        :returns:
+            A dictionary containing:
+            - `documents`: A list of `Document` objects with empty content and associated metadata.
+        """
+
+        documents = []
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+            document = Document(content=None, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/image/file_to_image.py

@@ -0,0 +1,150 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import mimetypes
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import component, logging
+from haystack.components.converters.image.image_utils import _encode_image_to_base64
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.dataclasses.image_content import ImageContent
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(
+    "The 'size' parameter is set. "
+    "Image resizing will be applied, which requires the Pillow library. "
+    "Run 'pip install pillow'"
+) as pillow_import:
+    import PIL  # noqa: F401
+
+logger = logging.getLogger(__name__)
+
+
+_EMPTY_BYTE_STRING = b""
+
+
+@component
+class ImageFileToImageContent:
+    """
+    Converts image files to ImageContent objects.
+
+    ### Usage example
+    ```python
+    from haystack.components.converters.image import ImageFileToImageContent
+
+    converter = ImageFileToImageContent()
+
+    sources = ["image.jpg", "another_image.png"]
+
+    image_contents = converter.run(sources=sources)["image_contents"]
+    print(image_contents)
+
+    # [ImageContent(base64_image='...',
+    #               mime_type='image/jpeg',
+    #               detail=None,
+    #               meta={'file_path': 'image.jpg'}),
+    #  ...]
+    ```
+    """
+
+    def __init__(
+        self, *, detail: Literal["auto", "high", "low"] | None = None, size: tuple[int, int] | None = None
+    ) -> None:
+        """
+        Create the ImageFileToImageContent component.
+
+        :param detail: Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+        :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+        """
+        self.detail = detail
+        self.size = size
+
+        if self.size is not None:
+            pillow_import.check()
+
+    @component.output_types(image_contents=list[ImageContent])
+    def run(
+        self,
+        sources: list[str | Path | ByteStream],
+        meta: dict[str, Any] | list[dict[str, Any]] | None = None,
+        *,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+    ) -> dict[str, list[ImageContent]]:
+        """
+        Converts files to ImageContent objects.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the ImageContent objects.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced ImageContent objects.
+            If it's a list, its length must match the number of sources as they're zipped together.
+            For ByteStream objects, their `meta` is added to the output ImageContent objects.
+        :param detail:
+            Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+            If not provided, the detail level will be the one set in the constructor.
+        :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+            If not provided, the size value will be the one set in the constructor.
+
+        :returns:
+            A dictionary with the following keys:
+            - `image_contents`: A list of ImageContent objects.
+        """
+        if not sources:
+            return {"image_contents": []}
+
+        resolved_detail = detail or self.detail
+        resolved_size = size or self.size
+
+        # Check import
+        if resolved_size:
+            pillow_import.check()
+
+        image_contents = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            if isinstance(source, str):
+                source = Path(source)
+
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            if bytestream.mime_type is None and isinstance(source, Path):
+                bytestream = replace(bytestream, mime_type=mimetypes.guess_type(source.as_posix())[0])
+
+            if bytestream.data == _EMPTY_BYTE_STRING:
+                logger.warning("File {source} is empty. Skipping it.", source=source)
+                continue
+
+            try:
+                inferred_mime_type, base64_image = _encode_image_to_base64(bytestream=bytestream, size=resolved_size)
+            except Exception as e:
+                logger.warning(
+                    "Could not convert file {source}. Skipping it. Error message: {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+            image_content = ImageContent(
+                base64_image=base64_image, mime_type=inferred_mime_type, meta=merged_metadata, detail=resolved_detail
+            )
+            image_contents.append(image_content)
+
+        return {"image_contents": image_contents}

haystack/components/converters/image/image_utils.py

@@ -0,0 +1,338 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import base64
+import mimetypes
+from collections import defaultdict
+from io import BytesIO
+from pathlib import Path
+from typing import TypedDict, Union
+
+from typing_extensions import NotRequired
+
+from haystack import logging
+from haystack.dataclasses import ByteStream, Document
+from haystack.dataclasses.image_content import IMAGE_MIME_TYPES, MIME_TO_FORMAT
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install pypdfium2'") as pypdfium2_import:
+    from pypdfium2 import PdfDocument
+
+with LazyImport("Run 'pip install pillow'") as pillow_import:
+    from PIL import Image as PILImage
+    from PIL.Image import Image
+    from PIL.ImageFile import ImageFile
+
+
+logger = logging.getLogger(__name__)
+
+
+def _encode_image_to_base64(bytestream: ByteStream, size: tuple[int, int] | None = None) -> tuple[str | None, str]:
+    """
+    Encode an image from a ByteStream into a base64-encoded string.
+
+    Optionally resize the image before encoding to improve performance for downstream processing.
+
+    :param bytestream: ByteStream containing the image data.
+    :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+        maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+        when working with models that have resolution constraints or when transmitting images to remote services.
+
+    :returns:
+        A tuple (mime_type, base64_str), where:
+        - mime_type (Optional[str]): The mime type of the encoded image, determined from the original data or image
+          content. Can be None if the mime type cannot be reliably identified.
+        - base64_str (str): The base64-encoded string representation of the (optionally resized) image.
+    """
+    if size is None:
+        if bytestream.mime_type is None:
+            logger.warning(
+                "No mime type provided for the image. "
+                "This may cause compatibility issues with downstream systems requiring a specific mime type. "
+                "Please provide a mime type for the image."
+            )
+        return bytestream.mime_type, base64.b64encode(bytestream.data).decode("utf-8")
+
+    # Check the import
+    pillow_import.check()
+
+    # Load the image
+    if bytestream.mime_type and bytestream.mime_type in MIME_TO_FORMAT:
+        formats = [MIME_TO_FORMAT[bytestream.mime_type]]
+    else:
+        formats = None
+    image: "ImageFile" = PILImage.open(BytesIO(bytestream.data), formats=formats)
+
+    # NOTE: We prefer the format returned by PIL
+    inferred_mime_type = image.get_format_mimetype() or bytestream.mime_type
+
+    # Downsize the image in place
+    if size is not None:
+        # Set reducing_gap=None to disable multi-step shrink; better quality.
+        # https://pillow.readthedocs.io/en/latest/reference/Image.html#PIL.Image.Image.thumbnail
+        image.thumbnail(size=size, reducing_gap=None)
+
+    # Convert the image to base64 string
+    if not inferred_mime_type:
+        logger.warning(
+            "Could not determine mime type for image. Defaulting to 'image/jpeg'. "
+            "Consider providing a mime_type parameter."
+        )
+        inferred_mime_type = "image/jpeg"
+    return inferred_mime_type, _encode_pil_image_to_base64(image=image, mime_type=inferred_mime_type)
+
+
+def _encode_pil_image_to_base64(image: Union["Image", "ImageFile"], mime_type: str = "image/jpeg") -> str:
+    """
+    Convert a PIL Image object to a base64-encoded string.
+
+    Automatically converts images with transparency to RGB if saving as JPEG.
+
+    :param image: A PIL Image or ImageFile object to encode.
+    :param mime_type: The MIME type to use when encoding the image. Defaults to "image/jpeg".
+    :returns:
+        Base64-encoded string representing the image.
+    """
+    # Check the import
+    pillow_import.check()
+
+    # Convert image to RGB if it has an alpha channel and we are saving as JPEG
+    if (mime_type == "image/jpeg" or mime_type == "image/jpg") and (
+        image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info)
+    ):
+        image = image.convert("RGB")
+
+    buffered = BytesIO()
+    form = MIME_TO_FORMAT.get(mime_type)
+    if form is None:
+        logger.warning("Could not determine format for mime type {mime_type}. Defaulting to JPEG.", mime_type=mime_type)
+        form = "JPEG"
+    image.save(buffered, format=form)
+    return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+
+def _convert_pdf_to_images(
+    *,
+    bytestream: ByteStream,
+    return_base64: bool = False,
+    page_range: list[int] | None = None,
+    size: tuple[int, int] | None = None,
+) -> list[tuple[int, "Image"]] | list[tuple[int, str]]:
+    """
+    Convert a PDF file into a list of PIL Image objects or base64-encoded images.
+
+    Checks PDF dimensions and adjusts size constraints based on aspect ratio.
+
+    :param bytestream: ByteStream object containing the PDF data
+    :param return_base64: If True, return base64-encoded images instead of PIL images.
+    :param page_range: List of page numbers and/or page ranges to convert to images. Page numbers start at 1.
+        If None, all pages in the PDF will be converted. Pages outside the valid range (1 to number of pages)
+        will be skipped with a warning. For example, page_range=[1, 3] will convert only the first and third
+        pages of the document. It also accepts printable range strings, e.g.:  ['1-3', '5', '8', '10-12']
+        will convert pages 1, 2, 3, 5, 8, 10, 11, 12.
+    :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+        maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+        when working with models that have resolution constraints or when transmitting images to remote services.
+
+    :returns:
+        A list of tuples, each tuple containing the page number and the PIL Image object or base64-encoded image string.
+    """
+
+    pypdfium2_import.check()
+    pillow_import.check()
+
+    try:
+        pdf = PdfDocument(BytesIO(bytestream.data))
+    except Exception as e:
+        logger.warning(
+            "Could not read PDF file {file_path}. Skipping it. Error: {error}",
+            file_path=bytestream.meta.get("file_path"),
+            error=e,
+        )
+        return []
+
+    num_pages = len(pdf)
+    if num_pages == 0:
+        logger.warning("PDF file is empty: {file_path}", file_path=bytestream.meta.get("file_path"))
+        pdf.close()
+        return []
+
+    all_pdf_images = []
+
+    resolved_page_range = page_range or range(1, num_pages + 1)
+
+    for page_number in resolved_page_range:
+        if page_number < 1 or page_number > num_pages:
+            logger.warning("Page {page_number} is out of range for the PDF file. Skipping it.", page_number=page_number)
+            continue
+
+        # Get dimensions of the page
+        page = pdf[max(page_number - 1, 0)]  # Adjust for 0-based indexing
+        _, _, width, height = page.get_mediabox()
+
+        target_resolution_dpi = 300.0
+
+        # From pypdfium2 docs: scale (float) – A factor scaling the number of pixels per PDF canvas unit. This defines
+        # the resolution of the image. To convert a DPI value to a scale factor, multiply it by the size of 1 canvas
+        # unit in inches (usually 1/72in).
+        # https://pypdfium2.readthedocs.io/en/stable/python_api.html#pypdfium2._helpers.page.PdfPage.render
+        target_scale = target_resolution_dpi / 72.0
+
+        # Calculate potential pixels for target_dpi
+        pixels_for_target_scale = width * height * target_scale**2
+
+        pil_max_pixels = PILImage.MAX_IMAGE_PIXELS or int(1024 * 1024 * 1024 // 4 // 3)
+        # 90% of PIL's default limit to prevent borderline cases
+        pixel_limit = pil_max_pixels * 0.9
+
+        scale = target_scale
+        if pixels_for_target_scale > pixel_limit:
+            logger.info(
+                "Large PDF detected ({pixels:.2f} pixels). Resizing the image to fit the pixel limit.",
+                pixels=pixels_for_target_scale,
+            )
+            scale = (pixel_limit / (width * height)) ** 0.5
+
+        pdf_bitmap = page.render(scale=scale)
+
+        image: "Image" = pdf_bitmap.to_pil()
+        pdf_bitmap.close()
+        if size is not None:
+            # Set reducing_gap=None to disable multi-step shrink; better quality.
+            # https://pillow.readthedocs.io/en/latest/reference/Image.html#PIL.Image.Image.thumbnail
+            image.thumbnail(size=size, reducing_gap=None)
+
+        all_pdf_images.append((page_number, image))
+
+    pdf.close()
+
+    if return_base64:
+        return [
+            (page_number, _encode_pil_image_to_base64(image, mime_type="image/jpeg"))
+            for page_number, image in all_pdf_images
+        ]
+
+    return all_pdf_images
+
+
+class _ImageSourceInfo(TypedDict):
+    path: Path
+    mime_type: str | None
+    page_number: NotRequired[int]  # Only present for PDF documents
+
+
+def _extract_image_sources_info(
+    documents: list[Document], file_path_meta_field: str, root_path: str
+) -> list[_ImageSourceInfo]:
+    """
+    Extracts the image source information from the documents.
+
+    :param documents: List of documents to extract image source information from.
+    :param file_path_meta_field: The metadata field in the Document that contains the file path to the image or PDF.
+    :param root_path: The root directory path where document files are located.
+
+    :returns:
+        A list of _ImageSourceInfo dictionaries, each containing the path and type of the image.
+        If the image is a PDF, the dictionary also contains the page number.
+    :raises ValueError: If the document is missing the file_path_meta_field key in its metadata, the file path is
+        invalid, the MIME type is not supported, or the page number is missing for a PDF document.
+    """
+    images_source_info: list[_ImageSourceInfo] = []
+    for doc in documents:
+        file_path = doc.meta.get(file_path_meta_field)
+        if file_path is None:
+            raise ValueError(
+                f"Document with ID '{doc.id}' is missing the '{file_path_meta_field}' key in its metadata."
+                f" Please ensure that the documents you are trying to convert have this key set."
+            )
+
+        resolved_file_path = Path(root_path, file_path)
+
+        # When root_path is set, ensure the resolved path stays within it to block path-traversal
+        # payloads (e.g. "../../etc/passwd") coming from document metadata.
+        if root_path:
+            resolved_file_path = resolved_file_path.resolve()
+            resolved_root = Path(root_path).resolve()
+            if not resolved_file_path.is_relative_to(resolved_root):
+                raise ValueError(
+                    f"Document with ID '{doc.id}' has a file path '{file_path}' that escapes the "
+                    f"configured root '{root_path}'. Resolved path: '{resolved_file_path}'."
+                )
+
+        if not resolved_file_path.is_file():
+            raise ValueError(
+                f"Document with ID '{doc.id}' has an invalid file path '{resolved_file_path}'. "
+                f"Please ensure that the documents you are trying to convert have valid file paths."
+            )
+
+        mime_type = doc.meta.get("mime_type") or mimetypes.guess_type(resolved_file_path)[0]
+        if mime_type not in IMAGE_MIME_TYPES:
+            raise ValueError(
+                f"Document with file path '{resolved_file_path}' has an unsupported MIME type '{mime_type}'. "
+                f"Please ensure that the documents you are trying to convert are of the supported "
+                f"types: {', '.join(IMAGE_MIME_TYPES)}."
+            )
+
+        image_info: _ImageSourceInfo = {"path": resolved_file_path, "mime_type": mime_type}
+
+        # If mimetype is PDF we also need the page number to be able to convert the right page
+        if mime_type == "application/pdf":
+            page_number = doc.meta.get("page_number")
+            if page_number is None:
+                raise ValueError(
+                    f"Document with ID '{doc.id}' comes from the PDF file '{resolved_file_path}' but is missing "
+                    f"the 'page_number' key in its metadata. Please ensure that PDF documents you are trying to "
+                    f"convert have this key set."
+                )
+            image_info["page_number"] = page_number
+
+        images_source_info.append(image_info)
+
+    return images_source_info
+
+
+class _PDFPageInfo(TypedDict):
+    doc_idx: int
+    path: Path
+    page_number: int
+
+
+def _batch_convert_pdf_pages_to_images(
+    *, pdf_page_infos: list[_PDFPageInfo], return_base64: bool = False, size: tuple[int, int] | None = None
+) -> dict[int, str] | dict[int, "Image"]:
+    """
+    Converts selected PDF pages to images, returning a mapping from document indices to images (PIL or base64).
+
+    Pages are grouped by file path to ensure each PDF is opened and processed only once for efficiency.
+
+    :param pdf_page_infos: List of _PDFPageInfo dictionaries with doc_idx, path, and page_number.
+    :param size: Optional tuple of width and height to resize the images to.
+    :param return_base64: If True, return base64 encoded images instead of PIL images.
+
+    :returns: Dictionary mapping document indices to images (PIL.Image or base64 string).
+    """
+    if not pdf_page_infos:
+        return {}
+
+    page_infos_by_pdf_path = defaultdict(list)
+    for page_info in pdf_page_infos:
+        page_infos_by_pdf_path[page_info["path"]].append(page_info)
+
+    converted_images_by_doc_index = {}
+
+    for pdf_path, page_infos_for_pdf in page_infos_by_pdf_path.items():
+        page_numbers_to_convert = [info["page_number"] for info in page_infos_for_pdf]
+        bytestream = ByteStream.from_file_path(pdf_path)
+
+        converted_pages = _convert_pdf_to_images(
+            bytestream=bytestream, return_base64=return_base64, page_range=page_numbers_to_convert, size=size
+        )
+
+        # Map results back to document indices
+        page_number_to_image = dict(converted_pages)
+        for page_info in page_infos_for_pdf:
+            converted_images_by_doc_index[page_info["doc_idx"]] = page_number_to_image[page_info["page_number"]]
+
+    # mypy is not able to infer that we match the declared return type
+    return converted_images_by_doc_index  # type: ignore[return-value]

haystack/components/converters/image/pdf_to_image.py

@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import component, logging
+from haystack.components.converters.image.image_utils import _convert_pdf_to_images, pillow_import, pypdfium2_import
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.dataclasses.image_content import ImageContent
+from haystack.utils import expand_page_range
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class PDFToImageContent:
+    """
+    Converts PDF files to ImageContent objects.
+
+    ### Usage example
+    ```python
+    from haystack.components.converters.image import PDFToImageContent
+
+    converter = PDFToImageContent()
+
+    sources = ["file.pdf", "another_file.pdf"]
+
+    image_contents = converter.run(sources=sources)["image_contents"]
+    print(image_contents)
+
+    # [ImageContent(base64_image='...',
+    #               mime_type='application/pdf',
+    #               detail=None,
+    #               meta={'file_path': 'file.pdf', 'page_number': 1}),
+    #  ...]
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+        page_range: list[str | int] | None = None,
+    ) -> None:
+        """
+        Create the PDFToImageContent component.
+
+        :param detail: Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+        :param size: If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+        :param page_range: List of page numbers and/or page ranges to convert to images. Page numbers start at 1.
+            If None, all pages in the PDF will be converted. Pages outside the valid range (1 to number of pages)
+            will be skipped with a warning. For example, page_range=[1, 3] will convert only the first and third
+            pages of the document. It also accepts printable range strings, e.g.:  ['1-3', '5', '8', '10-12']
+            will convert pages 1, 2, 3, 5, 8, 10, 11, 12.
+        """
+        self.detail = detail
+        self.size = size
+        self.page_range = page_range
+        pypdfium2_import.check()
+        pillow_import.check()
+
+    @component.output_types(image_contents=list[ImageContent])
+    def run(
+        self,
+        sources: list[str | Path | ByteStream],
+        meta: dict[str, Any] | list[dict[str, Any]] | None = None,
+        *,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+        page_range: list[str | int] | None = None,
+    ) -> dict[str, list[ImageContent]]:
+        """
+        Converts files to ImageContent objects.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the ImageContent objects.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced ImageContent objects.
+            If it's a list, its length must match the number of sources as they're zipped together.
+            For ByteStream objects, their `meta` is added to the output ImageContent objects.
+        :param detail:
+            Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
+            This will be passed to the created ImageContent objects.
+            If not provided, the detail level will be the one set in the constructor.
+        :param size:
+            If provided, resizes the image to fit within the specified dimensions (width, height) while
+            maintaining aspect ratio. This reduces file size, memory usage, and processing time, which is beneficial
+            when working with models that have resolution constraints or when transmitting images to remote services.
+            If not provided, the size value will be the one set in the constructor.
+        :param page_range:
+            List of page numbers and/or page ranges to convert to images. Page numbers start at 1.
+            If None, all pages in the PDF will be converted. Pages outside the valid range (1 to number of pages)
+            will be skipped with a warning. For example, page_range=[1, 3] will convert only the first and third
+            pages of the document. It also accepts printable range strings, e.g.:  ['1-3', '5', '8', '10-12']
+            will convert pages 1, 2, 3, 5, 8, 10, 11, 12.
+            If not provided, the page_range value will be the one set in the constructor.
+
+        :returns:
+            A dictionary with the following keys:
+            - `image_contents`: A list of ImageContent objects.
+        """
+        if not sources:
+            return {"image_contents": []}
+
+        resolved_detail = detail or self.detail
+        resolved_size = size or self.size
+        resolved_page_range = page_range or self.page_range
+
+        expanded_page_range = expand_page_range(resolved_page_range) if resolved_page_range else None
+
+        image_contents = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            if isinstance(source, str):
+                source = Path(source)
+
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                page_num_and_base64_images = _convert_pdf_to_images(
+                    bytestream=bytestream, page_range=expanded_page_range, size=resolved_size, return_base64=True
+                )
+            except Exception as e:
+                logger.warning(
+                    "Could not convert file {source}. Skipping it. Error message: {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            for page_number, image in page_num_and_base64_images:
+                per_page_metadata = {**merged_metadata, "page_number": page_number}
+                # we already know that image is a string because we set return_base64=True but mypy doesn't know that
+                assert isinstance(image, str)
+                image_contents.append(
+                    ImageContent(
+                        base64_image=image, mime_type="image/jpeg", meta=per_page_metadata, detail=resolved_detail
+                    )
+                )
+
+        return {"image_contents": image_contents}

haystack/components/converters/json.py

@@ -0,0 +1,289 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream, Document
+from haystack.lazy_imports import LazyImport
+
+logger = logging.getLogger(__name__)
+
+with LazyImport("Run 'pip install jq'") as jq_import:
+    import jq
+
+
+@component
+class JSONConverter:
+    """
+    Converts one or more JSON files into a text document.
+
+    ### Usage examples
+
+    ```python
+    import json
+
+    from haystack.components.converters import JSONConverter
+    from haystack.dataclasses import ByteStream
+
+    source = ByteStream.from_string(json.dumps({"text": "This is the content of my document"}))
+
+    converter = JSONConverter(content_key="text")
+    results = converter.run(sources=[source])
+    documents = results["documents"]
+    print(documents[0].content)
+    # 'This is the content of my document'
+    ```
+
+    Optionally, you can also provide a `jq_schema` string to filter the JSON source files and `extra_meta_fields`
+    to extract from the filtered data:
+
+    ```python
+    import json
+
+    from haystack.components.converters import JSONConverter
+    from haystack.dataclasses import ByteStream
+
+    data = {
+        "laureates": [
+            {
+                "firstname": "Enrico",
+                "surname": "Fermi",
+                "motivation": "for his demonstrations of the existence of new radioactive elements produced "
+                "by neutron irradiation, and for his related discovery of nuclear reactions brought about by"
+                " slow neutrons",
+            },
+            {
+                "firstname": "Rita",
+                "surname": "Levi-Montalcini",
+                "motivation": "for their discoveries of growth factors",
+            },
+        ],
+    }
+    source = ByteStream.from_string(json.dumps(data))
+    converter = JSONConverter(
+        jq_schema=".laureates[]", content_key="motivation", extra_meta_fields={"firstname", "surname"}
+    )
+
+    results = converter.run(sources=[source])
+    documents = results["documents"]
+    print(documents[0].content)
+    # 'for his demonstrations of the existence of new radioactive elements produced by
+    # neutron irradiation, and for his related discovery of nuclear reactions brought
+    # about by slow neutrons'
+
+    print(documents[0].meta)
+    # {'firstname': 'Enrico', 'surname': 'Fermi'}
+
+    print(documents[1].content)
+    # 'for their discoveries of growth factors'
+
+    print(documents[1].meta)
+    # {'firstname': 'Rita', 'surname': 'Levi-Montalcini'}
+    ```
+
+    """
+
+    def __init__(
+        self,
+        jq_schema: str | None = None,
+        content_key: str | None = None,
+        extra_meta_fields: set[str] | Literal["*"] | None = None,
+        store_full_path: bool = False,
+    ) -> None:
+        """
+        Creates a JSONConverter component.
+
+        An optional `jq_schema` can be provided to extract nested data in the JSON source files.
+        See the [official jq documentation](https://jqlang.github.io/jq/) for more info on the filters syntax.
+        If `jq_schema` is not set, whole JSON source files will be used to extract content.
+
+        Optionally, you can provide a `content_key` to specify which key in the extracted object must
+        be set as the document's content.
+
+        If both `jq_schema` and `content_key` are set, the component will search for the `content_key` in
+        the JSON object extracted by `jq_schema`. If the extracted data is not a JSON object, it will be skipped.
+
+        If only `jq_schema` is set, the extracted data must be a scalar value. If it's a JSON object or array,
+        it will be skipped.
+
+        If only `content_key` is set, the source JSON file must be a JSON object, else it will be skipped.
+
+        `extra_meta_fields` can either be set to a set of strings or a literal `"*"` string.
+        If it's a set of strings, it must specify fields in the extracted objects that must be set in
+        the extracted documents. If a field is not found, the meta value will be `None`.
+        If set to `"*"`, all fields that are not `content_key` found in the filtered JSON object will
+        be saved as metadata.
+
+        Initialization will fail if neither `jq_schema` nor `content_key` are set.
+
+        :param jq_schema:
+            Optional jq filter string to extract content.
+            If not specified, whole JSON object will be used to extract information.
+        :param content_key:
+            Optional key to extract document content.
+            If `jq_schema` is specified, the `content_key` will be extracted from that object.
+        :param extra_meta_fields:
+            An optional set of meta keys to extract from the content.
+            If `jq_schema` is specified, all keys will be extracted from that object.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        self._compiled_filter = None
+        if jq_schema:
+            jq_import.check()
+            self._compiled_filter = jq.compile(jq_schema)
+
+        self._jq_schema = jq_schema
+        self._content_key = content_key
+        self._meta_fields = extra_meta_fields
+        self._store_full_path = store_full_path
+
+        if self._compiled_filter is None and self._content_key is None:
+            msg = "No `jq_schema` nor `content_key` specified. Set either or both to extract data."
+            raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            jq_schema=self._jq_schema,
+            content_key=self._content_key,
+            extra_meta_fields=self._meta_fields,
+            store_full_path=self._store_full_path,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "JSONConverter":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, Any]]]:
+        """
+        Utility function to extract text and metadata from a JSON file.
+
+        :param source:
+            UTF-8 byte stream.
+        :returns:
+            Collection of text and metadata dict tuples, each corresponding
+            to a different document.
+        """
+        try:
+            file_content = source.data.decode("utf-8")
+        except UnicodeError as exc:
+            logger.warning(
+                "Failed to extract text from {source}. Skipping it. Error: {error}",
+                source=source.meta["file_path"],
+                error=exc,
+            )
+            return []
+
+        meta_fields = self._meta_fields or set()
+
+        if self._compiled_filter is not None:
+            try:
+                objects = list(self._compiled_filter.input_text(file_content))
+            except Exception as exc:
+                logger.warning(
+                    "Failed to extract text from {source}. Skipping it. Error: {error}",
+                    source=source.meta["file_path"],
+                    error=exc,
+                )
+                return []
+        else:
+            # We just load the whole file as JSON if the user didn't provide a jq filter.
+            # We put it in a list even if it's not to ease handling it later on.
+            objects = [json.loads(file_content)]
+
+        result = []
+        if self._content_key is not None:
+            for obj in objects:
+                if not isinstance(obj, dict):
+                    logger.warning("Expected a dictionary but got {obj}. Skipping it.", obj=obj)
+                    continue
+                if self._content_key not in obj:
+                    logger.warning(
+                        "'{content_key}' not found in {obj}. Skipping it.", content_key=self._content_key, obj=obj
+                    )
+                    continue
+
+                text = obj[self._content_key]
+                if isinstance(text, (dict, list)):
+                    logger.warning("Expected a scalar value but got {obj}. Skipping it.", obj=obj)
+                    continue
+
+                meta = {}
+                if meta_fields == "*":
+                    meta = {k: v for k, v in obj.items() if k != self._content_key}
+                else:
+                    for field in meta_fields:
+                        meta[field] = obj.get(field, None)
+                result.append((text, meta))
+        else:
+            for obj in objects:
+                if isinstance(obj, (dict, list)):
+                    logger.warning("Expected a scalar value but got {obj}. Skipping it.", obj=obj)
+                    continue
+                result.append((str(obj), {}))
+
+        return result
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, Any]:
+        """
+        Converts a list of JSON files to documents.
+
+        :param sources:
+            A list of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, the length of the list must match the number of sources.
+            If `sources` contain ByteStream objects, their `meta` will be added to the output documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: A list of created documents.
+        """
+        documents = []
+        meta_list = normalize_metadata(meta=meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as exc:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=exc)
+                continue
+
+            data = self._get_content_and_meta(bytestream)
+
+            for text, extra_meta in data:
+                merged_metadata = {**bytestream.meta, **metadata, **extra_meta}
+
+                if not self._store_full_path and (file_path := bytestream.meta.get("file_path")):
+                    merged_metadata["file_path"] = os.path.basename(file_path)
+                document = Document(content=text, meta=merged_metadata)
+                documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/markdown.py

@@ -0,0 +1,180 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+from tqdm import tqdm
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install markdown-it-py mdit_plain'") as markdown_conversion_imports:
+    from markdown_it import MarkdownIt
+    from mdit_plain.renderer import RendererPlain
+
+
+logger = logging.getLogger(__name__)
+
+_FRONTMATTER_PATTERN = re.compile(r"\A---[ \t]*\r?\n(?P<frontmatter>.*?)(?:\r?\n)---[ \t]*(?:\r?\n|$)", re.DOTALL)
+
+
+@component
+class MarkdownToDocument:
+    """
+    Converts a Markdown file into a text Document.
+
+    Usage example:
+
+    ```python
+    from haystack.components.converters import MarkdownToDocument
+    from datetime import datetime
+
+    converter = MarkdownToDocument()
+    results = converter.run(
+        sources=["test/test_files/markdown/sample.md"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+    print(documents[0].content)
+    # 'This is a text from the markdown file.'
+    ```
+    """
+
+    def __init__(
+        self,
+        table_to_single_line: bool = False,
+        progress_bar: bool = True,
+        store_full_path: bool = False,
+        encoding: str = "utf-8",
+        *,
+        extract_frontmatter: bool = False,
+    ) -> None:
+        """
+        Create a MarkdownToDocument component.
+
+        :param table_to_single_line:
+            If True converts table contents into a single line.
+        :param progress_bar:
+            If True shows a progress bar when running.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        :param encoding:
+            The default encoding to use when converting Markdown files. If the encoding is specified in the metadata
+            of a source ByteStream, it overrides this value.
+        :param extract_frontmatter:
+            If True, YAML frontmatter at the beginning of the Markdown file is
+            removed from the document content and added to the document metadata.
+        """
+        markdown_conversion_imports.check()
+
+        self.table_to_single_line = table_to_single_line
+        self.progress_bar = progress_bar
+        self.store_full_path = store_full_path
+        self.encoding = encoding
+        self.extract_frontmatter = extract_frontmatter
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, Any]:
+        """
+        Converts a list of Markdown files to Documents.
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: List of created Documents
+        """
+        parser = MarkdownIt(renderer_cls=RendererPlain)
+        if self.table_to_single_line:
+            parser.enable("table")
+
+        documents = []
+        meta_list = normalize_metadata(meta=meta, sources_count=len(sources))
+
+        for source, metadata in tqdm(
+            zip(sources, meta_list, strict=True),
+            total=len(sources),
+            desc="Converting markdown files to Documents",
+            disable=not self.progress_bar,
+        ):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                file_content = bytestream.data.decode(encoding)
+                file_content, frontmatter = self._extract_frontmatter(file_content, source)
+                text = parser.render(file_content)
+            except Exception as conversion_e:
+                logger.warning(
+                    "Failed to extract text from {source}. Skipping it. Error: {error}",
+                    source=source,
+                    error=conversion_e,
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **frontmatter, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}
+
+    def _extract_frontmatter(self, file_content: str, source: str | Path | ByteStream) -> tuple[str, dict[str, Any]]:
+        if not self.extract_frontmatter:
+            return file_content, {}
+
+        match = _FRONTMATTER_PATTERN.match(file_content)
+        if not match:
+            return file_content, {}
+
+        frontmatter_text = match.group("frontmatter")
+        try:
+            frontmatter = json.loads(json.dumps(yaml.safe_load(frontmatter_text), default=str)) or {}
+        except yaml.YAMLError as error:
+            logger.warning(
+                "Could not parse YAML frontmatter in {source}. Keeping it as content. Error: {error}",
+                source=source,
+                error=error,
+            )
+            return file_content, {}
+        except (TypeError, ValueError) as error:
+            logger.warning(
+                "Could not convert YAML frontmatter in {source}. Keeping it as content. Error: {error}",
+                source=source,
+                error=error,
+            )
+            return file_content, {}
+
+        if not isinstance(frontmatter, dict):
+            logger.warning(
+                "Ignoring YAML frontmatter in {source}: expected a mapping, got {kind}.",
+                source=source,
+                kind=type(frontmatter).__name__,
+            )
+            return file_content, {}
+
+        return file_content[match.end() :], frontmatter

haystack/components/converters/msg.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import io
+import os
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install python-oxmsg'") as oxmsg_import:
+    from oxmsg import Message, recipient
+
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class MSGToDocument:
+    """
+    Converts Microsoft Outlook .msg files into Haystack Documents.
+
+    This component extracts email metadata (such as sender, recipients, CC, BCC, subject) and body content from .msg
+    files and converts them into structured Haystack Documents. Additionally, any file attachments within the .msg
+    file are extracted as ByteStream objects.
+
+    ### Example Usage
+
+    ```python
+    from haystack.components.converters.msg import MSGToDocument
+    from datetime import datetime
+
+    converter = MSGToDocument()
+    results = converter.run(sources=["test/test_files/msg/sample.msg"], meta={"date_added": datetime.now().isoformat()})
+    documents = results["documents"]
+    attachments = results["attachments"]
+    print(documents[0].content)
+    ```
+    """
+
+    def __init__(self, store_full_path: bool = False) -> None:
+        """
+        Creates a MSGToDocument component.
+
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        oxmsg_import.check()
+        self.store_full_path = store_full_path
+
+    @staticmethod
+    def _is_encrypted(msg: "Message") -> bool:
+        """
+        Determines whether the provided MSG file is encrypted.
+
+        :param msg: The MSG file as a parsed Message object.
+        :returns: True if the MSG file is encrypted, otherwise False.
+        """
+        return "encrypted" in msg.message_headers.get("Content-Type", "")
+
+    @staticmethod
+    def _create_recipient_str(recip: "recipient.Recipient") -> str:
+        """
+        Formats a recipient's name and email into a single string.
+
+        :param recip: A recipient object extracted from the MSG file.
+        :returns: A formatted string combining the recipient's name and email address.
+        """
+        recip_str = ""
+        if recip.name != "":
+            recip_str += f"{recip.name} "
+        if recip.email_address != "":
+            recip_str += f"{recip.email_address}"
+        return recip_str
+
+    def _convert(self, file_content: io.BytesIO) -> tuple[str, list[ByteStream]]:
+        """
+        Converts the MSG file content into text and extracts any attachments.
+
+        :param file_content: The MSG file content as a binary stream.
+        :returns: A tuple containing the extracted email text and a list of ByteStream objects for attachments.
+        :raises ValueError: If the MSG file is encrypted and cannot be read.
+        """
+        msg = Message.load(file_content)
+        if self._is_encrypted(msg):
+            raise ValueError("The MSG file is encrypted and cannot be read.")
+
+        txt = ""
+
+        # Sender
+        if msg.sender is not None:
+            txt += f"From: {msg.sender}\n"
+
+        # To
+        recipients_str = ",".join(self._create_recipient_str(r) for r in msg.recipients)
+        if recipients_str != "":
+            txt += f"To: {recipients_str}\n"
+
+        # CC
+        cc_header = msg.message_headers.get("Cc") or msg.message_headers.get("CC")
+        if cc_header is not None:
+            txt += f"Cc: {cc_header}\n"
+
+        # BCC
+        bcc_header = msg.message_headers.get("Bcc") or msg.message_headers.get("BCC")
+        if bcc_header is not None:
+            txt += f"Bcc: {bcc_header}\n"
+
+        # Subject
+        if msg.subject != "":
+            txt += f"Subject: {msg.subject}\n"
+
+        # Body
+        if msg.body is not None:
+            txt += "\n" + msg.body
+
+        # attachments
+        attachments = [
+            ByteStream(
+                data=attachment.file_bytes, meta={"file_path": attachment.file_name}, mime_type=attachment.mime_type
+            )
+            for attachment in msg.attachments
+            if attachment.file_bytes is not None
+        ]
+
+        return txt, attachments
+
+    @component.output_types(documents=list[Document], attachments=list[ByteStream])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[Document] | list[ByteStream]]:
+        """
+        Converts MSG files to Documents.
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created Documents.
+            - `attachments`: Created ByteStream objects from file attachments.
+        """
+        if len(sources) == 0:
+            return {"documents": [], "attachments": []}
+
+        documents = []
+        all_attachments = []
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                text, attachments = self._convert(io.BytesIO(bytestream.data))
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and "file_path" in bytestream.meta:
+                merged_metadata["file_path"] = os.path.basename(bytestream.meta["file_path"])
+
+            documents.append(Document(content=text, meta=merged_metadata))
+            for attachment in attachments:
+                attachment_meta = {
+                    **merged_metadata,
+                    "parent_file_path": merged_metadata["file_path"],
+                    "file_path": attachment.meta["file_path"],
+                }
+                all_attachments.append(
+                    ByteStream(data=attachment.data, meta=attachment_meta, mime_type=attachment.mime_type)
+                )
+
+        return {"documents": documents, "attachments": all_attachments}

haystack/components/converters/multi_file_converter.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from haystack import Document, Pipeline, super_component
+from haystack.components.converters import (
+    CSVToDocument,
+    DOCXToDocument,
+    HTMLToDocument,
+    JSONConverter,
+    PPTXToDocument,
+    PyPDFToDocument,
+    TextFileToDocument,
+    XLSXToDocument,
+)
+from haystack.components.joiners import DocumentJoiner
+from haystack.components.routers import FileTypeRouter
+from haystack.dataclasses import ByteStream
+
+
+class ConverterMimeType(str, Enum):
+    CSV = "text/csv"
+    DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    HTML = "text/html"
+    JSON = "application/json"
+    MD = "text/markdown"
+    TEXT = "text/plain"
+    PDF = "application/pdf"
+    PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+
+@super_component
+class MultiFileConverter:
+    """
+    A file converter that handles conversion of multiple file types.
+
+    The MultiFileConverter handles the following file types:
+    - CSV
+    - DOCX
+    - HTML
+    - JSON
+    - MD
+    - TEXT
+    - PDF (no OCR)
+    - PPTX
+    - XLSX
+
+    Usage example:
+    ```
+    from haystack.super_components.converters import MultiFileConverter
+
+    converter = MultiFileConverter()
+    converter.run(sources=["test/test_files/txt/doc_1.txt", "test/test_files/pdf/sample_pdf_1.pdf"], meta={})
+    ```
+    """
+
+    def __init__(self, encoding: str = "utf-8", json_content_key: str = "content") -> None:
+        """
+        Initialize the MultiFileConverter.
+
+        :param encoding: The encoding to use when reading files.
+        :param json_content_key: The key to use in a content field in a document when converting JSON files.
+        """
+        self.encoding = encoding
+        self.json_content_key = json_content_key
+
+        # initialize components
+        router = FileTypeRouter(
+            mime_types=[mime_type.value for mime_type in ConverterMimeType],
+            # Ensure common extensions are registered. Tests on Windows fail otherwise.
+            additional_mimetypes={
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+                "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+            },
+        )
+
+        # Create pipeline and add components
+        pp = Pipeline()
+
+        pp.add_component("router", router)
+        pp.add_component("docx", DOCXToDocument(link_format="markdown"))
+        pp.add_component(
+            "html",
+            HTMLToDocument(
+                extraction_kwargs={"output_format": "markdown", "include_tables": True, "include_links": True}
+            ),
+        )
+        pp.add_component("json", JSONConverter(content_key=self.json_content_key))
+        pp.add_component("md", TextFileToDocument(encoding=self.encoding))
+        pp.add_component("text", TextFileToDocument(encoding=self.encoding))
+        pp.add_component("pdf", PyPDFToDocument())
+        pp.add_component("pptx", PPTXToDocument())
+        pp.add_component("xlsx", XLSXToDocument())
+        pp.add_component("joiner", DocumentJoiner())
+        pp.add_component("csv", CSVToDocument(encoding=self.encoding))
+
+        for mime_type in ConverterMimeType:
+            pp.connect(f"router.{mime_type.value}", str(mime_type).lower().rsplit(".", maxsplit=1)[-1])
+
+        pp.connect("docx.documents", "joiner.documents")
+        pp.connect("html.documents", "joiner.documents")
+        pp.connect("json.documents", "joiner.documents")
+        pp.connect("md.documents", "joiner.documents")
+        pp.connect("text.documents", "joiner.documents")
+        pp.connect("pdf.documents", "joiner.documents")
+        pp.connect("pptx.documents", "joiner.documents")
+
+        pp.connect("csv.documents", "joiner.documents")
+        pp.connect("xlsx.documents", "joiner.documents")
+
+        self.pipeline = pp
+        self.output_mapping = {
+            "joiner.documents": "documents",
+            "router.unclassified": "unclassified",
+            "router.failed": "failed",
+        }
+        self.input_mapping = {"sources": ["router.sources"], "meta": ["router.meta"]}
+
+    if TYPE_CHECKING:
+        # fake method, never executed, but static analyzers will not complain about missing method
+        def run(  # noqa: D102
+            self, *, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+        ) -> dict[str, list[Document]]:  # noqa: D102
+            ...
+
+        def warm_up(self) -> None:  # noqa: D102
+            ...

haystack/components/converters/output_adapter.py

@@ -0,0 +1,179 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import ast
+import contextlib
+from collections.abc import Callable
+from typing import Any, TypeAlias
+
+import jinja2.runtime
+from jinja2 import TemplateSyntaxError
+from jinja2.nativetypes import NativeEnvironment
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.utils import deserialize_callable, deserialize_type, serialize_callable, serialize_type
+from haystack.utils.jinja2_extensions import _extract_template_variables_and_assignments
+
+logger = logging.getLogger(__name__)
+
+
+class OutputAdaptationException(Exception):
+    """Exception raised when there is an error during output adaptation."""
+
+
+@component
+class OutputAdapter:
+    """
+    Adapts output of a Component using Jinja templates.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.converters import OutputAdapter
+
+    adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+    documents = [Document(content="Test content")]
+    result = adapter.run(documents=documents)
+
+    assert result["output"] == "Test content"
+    ```
+    """
+
+    def __init__(
+        self,
+        template: str,
+        output_type: TypeAlias,
+        custom_filters: dict[str, Callable] | None = None,
+        unsafe: bool = False,
+    ) -> None:
+        """
+        Create an OutputAdapter component.
+
+        :param template:
+            A Jinja template that defines how to adapt the input data.
+            The variables in the template define the input of this instance.
+            e.g.
+            With this template:
+            ```
+            {{ documents[0].content }}
+            ```
+            The Component input will be `documents`.
+        :param output_type:
+            The type of output this instance will return.
+        :param custom_filters:
+            A dictionary of custom Jinja filters used in the template.
+        :param unsafe:
+            Enable execution of arbitrary code in the Jinja template.
+            This should only be used if you trust the source of the template as it can be lead to remote code execution.
+        """
+        self.custom_filters = {**(custom_filters or {})}
+        input_types: set[str] = set()
+
+        self._unsafe = unsafe
+
+        if self._unsafe:
+            msg = (
+                "Unsafe mode is enabled. This allows execution of arbitrary code in the Jinja template. "
+                "Use this only if you trust the source of the template."
+            )
+            logger.warning(msg)
+        self._env = (
+            NativeEnvironment() if self._unsafe else SandboxedEnvironment(undefined=jinja2.runtime.StrictUndefined)
+        )
+
+        try:
+            self._env.parse(template)  # Validate template syntax
+            self.template = template
+        except TemplateSyntaxError as e:
+            raise ValueError(f"Invalid Jinja template '{template}': {e}") from e
+
+        for name, filter_func in self.custom_filters.items():
+            self._env.filters[name] = filter_func
+
+        # b) extract variables in the template
+        assigned_variables, template_variables = _extract_template_variables_and_assignments(
+            env=self._env, template=self.template
+        )
+        route_input_names = template_variables - assigned_variables
+        input_types.update(route_input_names)
+
+        # the env is not needed, discarded automatically
+        component.set_input_types(self, **dict.fromkeys(input_types, Any))
+        component.set_output_types(self, output=output_type)
+        self.output_type = output_type
+
+    def run(self, **kwargs: Any) -> dict[str, Any]:
+        """
+        Renders the Jinja template with the provided inputs.
+
+        :param kwargs:
+            Must contain all variables used in the `template` string.
+        :returns:
+            A dictionary with the following keys:
+            - `output`: Rendered Jinja template.
+
+        :raises OutputAdaptationException: If template rendering fails.
+        """
+        # check if kwargs are empty
+        if not kwargs:
+            raise ValueError("No input data provided for output adaptation")
+        for name, filter_func in self.custom_filters.items():
+            self._env.filters[name] = filter_func
+        adapted_outputs = {}
+        try:
+            adapted_output_template = self._env.from_string(self.template)
+            output_result = adapted_output_template.render(**kwargs)
+            if isinstance(output_result, jinja2.runtime.Undefined):
+                raise OutputAdaptationException(f"Undefined variable in the template {self.template}; kwargs: {kwargs}")  # noqa: TRY301
+
+            # We suppress the exception in case the output is already a string, otherwise
+            # we try to evaluate it and would fail.
+            # This must be done cause the output could be different literal structures.
+            # This doesn't support any user types.
+            with contextlib.suppress(Exception):
+                if not self._unsafe:
+                    output_result = ast.literal_eval(output_result)
+
+            adapted_outputs["output"] = output_result
+        except Exception as e:
+            raise OutputAdaptationException(f"Error adapting {self.template} with {kwargs}: {e}") from e
+        return adapted_outputs
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        se_filters = {name: serialize_callable(filter_func) for name, filter_func in self.custom_filters.items()}
+        return default_to_dict(
+            self,
+            template=self.template,
+            output_type=serialize_type(self.output_type),
+            custom_filters=se_filters,
+            unsafe=self._unsafe,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OutputAdapter":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        init_params = data.get("init_parameters", {})
+        init_params["output_type"] = deserialize_type(init_params["output_type"])
+
+        custom_filters = init_params.get("custom_filters", {})
+        if custom_filters:
+            init_params["custom_filters"] = {
+                name: deserialize_callable(filter_func) if filter_func else None
+                for name, filter_func in custom_filters.items()
+            }
+        return default_from_dict(cls, data)

haystack/components/converters/pdfminer.py

@@ -0,0 +1,228 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import io
+import os
+import re
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install pdfminer.six'") as pdfminer_import:
+    from pdfminer.high_level import extract_pages
+    from pdfminer.layout import LAParams, LTTextContainer
+
+logger = logging.getLogger(__name__)
+
+CID_PATTERN = r"\(cid:\d+\)"  # regex pattern to detect CID characters
+
+
+@component
+class PDFMinerToDocument:
+    """
+    Converts PDF files to Documents.
+
+    Uses `pdfminer` compatible converters to convert PDF files to Documents. https://pdfminersix.readthedocs.io/en/latest/
+
+    Usage example:
+
+    ```python
+    from haystack.components.converters.pdfminer import PDFMinerToDocument
+    from datetime import datetime
+
+    converter = PDFMinerToDocument()
+    results = converter.run(
+        sources=["test/test_files/pdf/sample_pdf_1.pdf"], meta={"date_added": datetime.now().isoformat()}
+    )
+
+    print(results["documents"][0].content)
+    # >> 'This is a text from the PDF file.'
+    ```
+    """
+
+    def __init__(
+        self,
+        line_overlap: float = 0.5,
+        char_margin: float = 2.0,
+        line_margin: float = 0.5,
+        word_margin: float = 0.1,
+        boxes_flow: float | None = 0.5,
+        detect_vertical: bool = True,
+        all_texts: bool = False,
+        store_full_path: bool = False,
+    ) -> None:
+        """
+        Create a PDFMinerToDocument component.
+
+        :param line_overlap:
+            This parameter determines whether two characters are considered to be on
+            the same line based on the amount of overlap between them.
+            The overlap is calculated relative to the minimum height of both characters.
+        :param char_margin:
+            Determines whether two characters are part of the same line based on the distance between them.
+            If the distance is less than the margin specified, the characters are considered to be on the same line.
+            The margin is calculated relative to the width of the character.
+        :param word_margin:
+            Determines whether two characters on the same line are part of the same word
+            based on the distance between them. If the distance is greater than the margin specified,
+            an intermediate space will be added between them to make the text more readable.
+            The margin is calculated relative to the width of the character.
+        :param line_margin:
+            This parameter determines whether two lines are part of the same paragraph based on
+            the distance between them. If the distance is less than the margin specified,
+            the lines are considered to be part of the same paragraph.
+            The margin is calculated relative to the height of a line.
+        :param boxes_flow:
+            This parameter determines the importance of horizontal and vertical position when
+            determining the order of text boxes. A value between -1.0 and +1.0 can be set,
+            with -1.0 indicating that only horizontal position matters and +1.0 indicating
+            that only vertical position matters. Setting the value to 'None' will disable advanced
+            layout analysis, and text boxes will be ordered based on the position of their bottom left corner.
+        :param detect_vertical:
+            This parameter determines whether vertical text should be considered during layout analysis.
+        :param all_texts:
+            If layout analysis should be performed on text in figures.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+
+        pdfminer_import.check()
+
+        self.layout_params = LAParams(
+            line_overlap=line_overlap,
+            char_margin=char_margin,
+            line_margin=line_margin,
+            word_margin=word_margin,
+            boxes_flow=boxes_flow,
+            detect_vertical=detect_vertical,
+            all_texts=all_texts,
+        )
+        self.store_full_path = store_full_path
+        self.cid_pattern = re.compile(CID_PATTERN)
+
+    @staticmethod
+    def _converter(lt_page_objs: Iterator) -> str:
+        """
+        Extracts text from PDF pages then converts the text into a single str
+
+        :param lt_page_objs:
+            Python generator that yields PDF pages.
+
+        :returns:
+            PDF text converted to single str
+        """
+        pages = []
+        for page in lt_page_objs:
+            text = ""
+            for container in page:
+                # Keep text only
+                if isinstance(container, LTTextContainer):
+                    container_text = container.get_text()
+                    if container_text:
+                        text += "\n\n"
+                    text += container_text
+            pages.append(text)
+
+        # Add a page delimiter
+        return "\f".join(pages)
+
+    def detect_undecoded_cid_characters(self, text: str) -> dict[str, Any]:
+        """
+        Look for character sequences of CID, i.e.: characters that haven't been properly decoded from their CID format.
+
+        This is useful to detect if the text extractor is not able to extract the text correctly, e.g. if the PDF uses
+        non-standard fonts.
+
+        A PDF font may include a ToUnicode map (mapping from character code to Unicode) to support operations like
+        searching strings or copy & paste in a PDF viewer. This map immediately provides the mapping the text extractor
+        needs. If that map is not available the text extractor cannot decode the CID characters and will return them
+        as is.
+
+        see: https://pdfminersix.readthedocs.io/en/latest/faq.html#why-are-there-cid-x-values-in-the-textual-output
+
+        :param text: The text to check for undecoded CID characters
+        :returns:
+            A dictionary containing detection results
+        """
+
+        matches = re.findall(self.cid_pattern, text)
+        total_chars = len(text)
+        cid_chars = sum(len(match) for match in matches)
+        percentage = (cid_chars / total_chars * 100) if total_chars > 0 else 0
+
+        return {"total_chars": total_chars, "cid_chars": cid_chars, "percentage": round(percentage, 2)}
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, Any]:
+        """
+        Converts PDF files to Documents.
+
+        :param sources:
+            List of PDF file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created Documents
+        """
+        documents = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                pages = extract_pages(io.BytesIO(bytestream.data), laparams=self.layout_params)
+                text = self._converter(pages)
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
+                )
+                continue
+
+            if text is None or text.strip() == "":
+                logger.warning(
+                    "PDFMinerToDocument could not extract text from the file {source}. Returning an empty document.",
+                    source=source,
+                )
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+
+            analysis = self.detect_undecoded_cid_characters(text)
+
+            if analysis["percentage"] > 0:
+                logger.warning(
+                    "Detected {cid_chars} undecoded CID characters in {total_chars} characters"
+                    " ({percentage}%) in {source}.",
+                    cid_chars=analysis["cid_chars"],
+                    total_chars=analysis["total_chars"],
+                    percentage=analysis["percentage"],
+                    source=source,
+                )
+
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/pptx.py

@@ -0,0 +1,158 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import io
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import Document, component, default_to_dict, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install python-pptx'") as pptx_import:
+    from pptx import Presentation
+    from pptx.text.text import _Paragraph
+
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class PPTXToDocument:
+    """
+    Converts PPTX files to Documents.
+
+    Usage example:
+
+    ```python
+    from haystack.components.converters.pptx import PPTXToDocument
+    from datetime import datetime
+
+    converter = PPTXToDocument()
+    results = converter.run(
+        sources=["test/test_files/pptx/sample_pptx.pptx"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> 'This is the text from the PPTX file.'
+    ```
+    """
+
+    def __init__(
+        self, store_full_path: bool = False, link_format: Literal["markdown", "plain", "none"] = "none"
+    ) -> None:
+        """
+        Create a PPTXToDocument component.
+
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        :param link_format: The format for link output. Possible options:
+            - `"markdown"`: `[text](url)`
+            - `"plain"`: `text (url)`
+            - `"none"`: Only the text is extracted, link addresses are ignored.
+        """
+        pptx_import.check()
+        if link_format not in ("markdown", "plain", "none"):
+            msg = f"Unknown link format '{link_format}'. Supported formats are: 'markdown', 'plain', 'none'"
+            raise ValueError(msg)
+        self.link_format = link_format
+        self.store_full_path = store_full_path
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, link_format=self.link_format, store_full_path=self.store_full_path)
+
+    def _convert(self, file_content: io.BytesIO) -> str:
+        """
+        Converts the PPTX file to text.
+        """
+        pptx_presentation = Presentation(file_content)
+        text_all_slides = []
+        for slide in pptx_presentation.slides:
+            text_on_slide = []
+            for shape in slide.shapes:
+                if shape.has_text_frame:
+                    paragraphs = []
+                    for paragraph in shape.text_frame.paragraphs:
+                        paragraphs.append(self._process_paragraph(paragraph))
+                    text_on_slide.append("\n".join(paragraphs))
+                elif hasattr(shape, "text"):
+                    text_on_slide.append(shape.text)
+            text_all_slides.append("\n".join(text_on_slide))
+        return "\f".join(text_all_slides)
+
+    def _process_paragraph(self, paragraph: "_Paragraph") -> str:
+        """
+        Processes a paragraph and formats hyperlinks according to the specified link format.
+
+        :param paragraph: The PPTX paragraph to process.
+        :returns: A string with links formatted according to the specified format.
+        """
+        if self.link_format == "none":
+            return paragraph.text
+        parts = []
+        for run in paragraph.runs:
+            if run.hyperlink and run.hyperlink.address:
+                if self.link_format == "markdown":
+                    parts.append(f"[{run.text}]({run.hyperlink.address})")
+                else:
+                    parts.append(f"{run.text} ({run.hyperlink.address})")
+            else:
+                parts.append(run.text)
+        return "".join(parts)
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, Any]:
+        """
+        Converts PPTX files to Documents.
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the Documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced Documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created Documents
+        """
+        documents = []
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                text = self._convert(io.BytesIO(bytestream.data))
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+            documents.append(Document(content=text, meta=merged_metadata))
+
+        return {"documents": documents}

haystack/components/converters/pypdf.py

@@ -0,0 +1,228 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import io
+import os
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install pypdf'") as pypdf_import:
+    from pypdf import PdfReader
+
+
+logger = logging.getLogger(__name__)
+
+
+class PyPDFExtractionMode(Enum):
+    """
+    The mode to use for extracting text from a PDF.
+    """
+
+    PLAIN = "plain"
+    LAYOUT = "layout"
+
+    def __str__(self) -> str:
+        """
+        Convert a PyPDFExtractionMode enum to a string.
+        """
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "PyPDFExtractionMode":
+        """
+        Convert a string to a PyPDFExtractionMode enum.
+        """
+        enum_map = {e.value: e for e in PyPDFExtractionMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown extraction mode '{string}'. Supported modes are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class PyPDFToDocument:
+    """
+    Converts PDF files to documents your pipeline can query.
+
+    This component uses the PyPDF library.
+    You can attach metadata to the resulting documents.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.converters.pypdf import PyPDFToDocument
+    from datetime import datetime
+
+    converter = PyPDFToDocument()
+    results = converter.run(
+        sources=["test/test_files/pdf/sample_pdf_1.pdf"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> 'This is a text from the PDF file.'
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        extraction_mode: str | PyPDFExtractionMode = PyPDFExtractionMode.PLAIN,
+        plain_mode_orientations: tuple = (0, 90, 180, 270),
+        plain_mode_space_width: float = 200.0,
+        layout_mode_space_vertically: bool = True,
+        layout_mode_scale_weight: float = 1.25,
+        layout_mode_strip_rotated: bool = True,
+        layout_mode_font_height_weight: float = 1.0,
+        store_full_path: bool = False,
+    ) -> None:
+        """
+        Create an PyPDFToDocument component.
+
+        :param extraction_mode:
+            The mode to use for extracting text from a PDF.
+            Layout mode is an experimental mode that adheres to the rendered layout of the PDF.
+        :param plain_mode_orientations:
+            Tuple of orientations to look for when extracting text from a PDF in plain mode.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.LAYOUT`.
+        :param plain_mode_space_width:
+            Forces default space width if not extracted from font.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.LAYOUT`.
+        :param layout_mode_space_vertically:
+            Whether to include blank lines inferred from y distance + font height.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
+        :param layout_mode_scale_weight:
+            Multiplier for string length when calculating weighted average character width.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
+        :param layout_mode_strip_rotated:
+            Layout mode does not support rotated text. Set to `False` to include rotated text anyway.
+            If rotated text is discovered, layout will be degraded and a warning will be logged.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
+        :param layout_mode_font_height_weight:
+            Multiplier for font height when calculating blank line height.
+            Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        pypdf_import.check()
+
+        self.store_full_path = store_full_path
+
+        if isinstance(extraction_mode, str):
+            extraction_mode = PyPDFExtractionMode.from_str(extraction_mode)
+        self.extraction_mode = extraction_mode
+        self.plain_mode_orientations = plain_mode_orientations
+        self.plain_mode_space_width = plain_mode_space_width
+        self.layout_mode_space_vertically = layout_mode_space_vertically
+        self.layout_mode_scale_weight = layout_mode_scale_weight
+        self.layout_mode_strip_rotated = layout_mode_strip_rotated
+        self.layout_mode_font_height_weight = layout_mode_font_height_weight
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            extraction_mode=str(self.extraction_mode),
+            plain_mode_orientations=self.plain_mode_orientations,
+            plain_mode_space_width=self.plain_mode_space_width,
+            layout_mode_space_vertically=self.layout_mode_space_vertically,
+            layout_mode_scale_weight=self.layout_mode_scale_weight,
+            layout_mode_strip_rotated=self.layout_mode_strip_rotated,
+            layout_mode_font_height_weight=self.layout_mode_font_height_weight,
+            store_full_path=self.store_full_path,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "PyPDFToDocument":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary with serialized data.
+
+        :returns:
+            Deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    def _default_convert(self, reader: "PdfReader") -> str:
+        texts = []
+        for page in reader.pages:
+            extracted_text = page.extract_text(
+                orientations=self.plain_mode_orientations,
+                extraction_mode=self.extraction_mode.value,
+                space_width=self.plain_mode_space_width,
+                layout_mode_space_vertically=self.layout_mode_space_vertically,
+                layout_mode_scale_weight=self.layout_mode_scale_weight,
+                layout_mode_strip_rotated=self.layout_mode_strip_rotated,
+                layout_mode_font_height_weight=self.layout_mode_font_height_weight,
+            )
+            texts.append(extracted_text)
+        return "\f".join(texts)
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Converts PDF files to documents.
+
+        :param sources:
+            List of file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, its length must match the number of sources, as they are zipped together.
+            For ByteStream objects, their `meta` is added to the output documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: A list of converted documents.
+        """
+        documents = []
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                pdf_reader = PdfReader(io.BytesIO(bytestream.data))
+                text = self._default_convert(pdf_reader)
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
+                )
+                continue
+
+            if text is None or text.strip() == "":
+                logger.warning(
+                    "PyPDFToDocument could not extract text from the file {source}. Returning an empty document.",
+                    source=source,
+                )
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/txt.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class TextFileToDocument:
+    """
+    Converts text files to documents your pipeline can query.
+
+    By default, it uses UTF-8 encoding when converting files but
+    you can also set custom encoding.
+    It can attach metadata to the resulting documents.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.converters.txt import TextFileToDocument
+
+    converter = TextFileToDocument()
+    results = converter.run(sources=["test/test_files/txt/doc_1.txt"])
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> 'This is the content from the txt file.'
+    ```
+    """
+
+    def __init__(self, encoding: str = "utf-8", store_full_path: bool = False) -> None:
+        """
+        Creates a TextFileToDocument component.
+
+        :param encoding:
+            The encoding of the text files to convert.
+            If the encoding is specified in the metadata of a source ByteStream,
+            it overrides this value.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        self.encoding = encoding
+        self.store_full_path = store_full_path
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Converts text files to documents.
+
+        :param sources:
+            List of text file paths or ByteStream objects to convert.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, its length must match the number of sources as they're zipped together.
+            For ByteStream objects, their `meta` is added to the output documents.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: A list of converted documents.
+        """
+        documents = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+            try:
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                text = bytestream.data.decode(encoding)
+            except Exception as e:
+                logger.warning(
+                    "Could not convert file {source}. Skipping it. Error message: {error}", source=source, error=e
+                )
+                continue
+
+            merged_metadata = {**bytestream.meta, **metadata}
+
+            if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
+                merged_metadata["file_path"] = os.path.basename(file_path)
+            document = Document(content=text, meta=merged_metadata)
+            documents.append(document)
+
+        return {"documents": documents}

haystack/components/converters/utils.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+from typing import Any
+
+from haystack.dataclasses import ByteStream
+
+
+def get_bytestream_from_source(source: str | Path | ByteStream, guess_mime_type: bool = False) -> ByteStream:
+    """
+    Creates a ByteStream object from a source.
+
+    :param source:
+        A source to convert to a ByteStream. Can be a string (path to a file), a Path object, or a ByteStream.
+    :param guess_mime_type:
+        Whether to guess the mime type from the file.
+    :return:
+        A ByteStream object.
+    """
+
+    if isinstance(source, ByteStream):
+        return source
+    if isinstance(source, (str, Path)):
+        bs = ByteStream.from_file_path(Path(source), guess_mime_type=guess_mime_type)
+        bs.meta["file_path"] = str(source)
+        return bs
+    raise ValueError(f"Unsupported source type {type(source)}")
+
+
+def normalize_metadata(meta: dict[str, Any] | list[dict[str, Any]] | None, sources_count: int) -> list[dict[str, Any]]:
+    """
+    Normalize the metadata input for a converter.
+
+    Given all the possible value of the meta input for a converter (None, dictionary or list of dicts),
+    makes sure to return a list of dictionaries of the correct length for the converter to use.
+
+    :param meta: the meta input of the converter, as-is
+    :param sources_count: the number of sources the converter received
+    :returns: a list of dictionaries of the make length as the sources list
+    """
+    if meta is None:
+        return [{}] * sources_count
+    if isinstance(meta, dict):
+        return [meta] * sources_count
+    if isinstance(meta, list):
+        if sources_count != len(meta):
+            raise ValueError("The length of the metadata list must match the number of sources.")
+        return meta
+    raise ValueError("meta must be either None, a dictionary or a list of dictionaries.")

haystack/components/converters/xlsx.py

@@ -0,0 +1,236 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import io
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import Document, component, logging
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+
+logger = logging.getLogger(__name__)
+
+with LazyImport("Run 'pip install pandas openpyxl'") as pandas_xlsx_import:
+    import openpyxl
+    import pandas as pd
+
+with LazyImport("Run 'pip install tabulate'") as tabulate_import:
+    from tabulate import tabulate  # noqa: F401 # the library is used but not directly referenced
+
+
+@component
+class XLSXToDocument:
+    """
+    Converts XLSX (Excel) files into Documents.
+
+    Supports reading data from specific sheets or all sheets in the Excel file. If all sheets are read, a Document is
+    created for each sheet. The content of the Document is the table which can be saved in CSV or Markdown format.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.converters.xlsx import XLSXToDocument
+    from datetime import datetime
+
+    converter = XLSXToDocument()
+    results = converter.run(
+        sources=["test/test_files/xlsx/basic_tables_two_sheets.xlsx"], meta={"date_added": datetime.now().isoformat()}
+    )
+    documents = results["documents"]
+
+    print(documents[0].content)
+    # >> ",A,B\\n1,col_a,col_b\\n2,1.5,test\\n"
+    ```
+    """
+
+    def __init__(
+        self,
+        table_format: Literal["csv", "markdown"] = "csv",
+        sheet_name: str | int | list[str | int] | None = None,
+        read_excel_kwargs: dict[str, Any] | None = None,
+        table_format_kwargs: dict[str, Any] | None = None,
+        *,
+        link_format: Literal["markdown", "plain", "none"] = "none",
+        store_full_path: bool = False,
+    ) -> None:
+        """
+        Creates a XLSXToDocument component.
+
+        :param table_format: The format to convert the Excel file to.
+        :param sheet_name: The name of the sheet to read. If None, all sheets are read.
+        :param read_excel_kwargs: Additional arguments to pass to `pandas.read_excel`.
+            See https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html#pandas-read-excel
+        :param table_format_kwargs: Additional keyword arguments to pass to the table format function.
+            - If `table_format` is "csv", these arguments are passed to `pandas.DataFrame.to_csv`.
+              See https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html#pandas-dataframe-to-csv
+            - If `table_format` is "markdown", these arguments are passed to `pandas.DataFrame.to_markdown`.
+              See https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html#pandas-dataframe-to-markdown
+        :param link_format: The format for link output. Possible options:
+            - `"markdown"`: `[text](url)`
+            - `"plain"`: `text (url)`
+            - `"none"`: Only the text is extracted, link addresses are ignored.
+        :param store_full_path:
+            If True, the full path of the file is stored in the metadata of the document.
+            If False, only the file name is stored.
+        """
+        pandas_xlsx_import.check()
+        self.table_format = table_format
+        if table_format not in ["csv", "markdown"]:
+            raise ValueError(f"Unsupported export format: {table_format}. Choose either 'csv' or 'markdown'.")
+        if link_format not in ("markdown", "plain", "none"):
+            msg = f"Unknown link format '{link_format}'. Supported formats are: 'markdown', 'plain', 'none'"
+            raise ValueError(msg)
+        if table_format == "markdown":
+            tabulate_import.check()
+        self.link_format = link_format
+        self.sheet_name = sheet_name
+        self.read_excel_kwargs = read_excel_kwargs or {}
+        self.table_format_kwargs = table_format_kwargs or {}
+        self.store_full_path = store_full_path
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Converts a XLSX file to a Document.
+
+        :param sources:
+            List of file paths or ByteStream objects.
+        :param meta:
+            Optional metadata to attach to the documents.
+            This value can be either a list of dictionaries or a single dictionary.
+            If it's a single dictionary, its content is added to the metadata of all produced documents.
+            If it's a list, the length of the list must match the number of sources, because the two lists will
+            be zipped.
+            If `sources` contains ByteStream objects, their `meta` will be added to the output documents.
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Created documents
+        """
+        documents = []
+
+        meta_list = normalize_metadata(meta, sources_count=len(sources))
+
+        for source, metadata in zip(sources, meta_list, strict=True):
+            try:
+                bytestream = get_bytestream_from_source(source)
+            except Exception as e:
+                logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
+                continue
+
+            try:
+                tables, tables_metadata = self._extract_tables(bytestream)
+            except Exception as e:
+                logger.warning(
+                    "Could not read {source} and convert it to a Document, skipping. Error: {error}",
+                    source=source,
+                    error=e,
+                )
+                continue
+
+            # Loop over tables and create a Document for each table
+            for table, excel_metadata in zip(tables, tables_metadata, strict=True):
+                merged_metadata = {**bytestream.meta, **metadata, **excel_metadata}
+
+                if not self.store_full_path and "file_path" in bytestream.meta:
+                    file_path = bytestream.meta["file_path"]
+                    merged_metadata["file_path"] = os.path.basename(file_path)
+
+                document = Document(content=table, meta=merged_metadata)
+                documents.append(document)
+
+        return {"documents": documents}
+
+    @staticmethod
+    def _generate_excel_column_names(n_cols: int) -> list[str]:
+        result = []
+        for i in range(n_cols):
+            col_name = ""
+            num = i
+            while num >= 0:
+                col_name = chr(num % 26 + 65) + col_name
+                num = num // 26 - 1
+            result.append(col_name)
+        return result
+
+    def _extract_tables(self, bytestream: ByteStream) -> tuple[list[str], list[dict]]:
+        """
+        Extract tables from an Excel file.
+        """
+        file_bytes = io.BytesIO(bytestream.data)
+        resolved_read_excel_kwargs = {
+            **self.read_excel_kwargs,
+            "sheet_name": self.sheet_name,
+            "header": None,  # Don't assign any pandas column labels
+            "engine": "openpyxl",  # Use openpyxl as the engine to read the Excel file
+        }
+        sheet_to_dataframe = pd.read_excel(io=file_bytes, **resolved_read_excel_kwargs)
+        if isinstance(sheet_to_dataframe, pd.DataFrame):
+            sheet_to_dataframe = {self.sheet_name: sheet_to_dataframe}
+
+        # If link extraction is enabled, load the workbook with openpyxl to read hyperlinks
+        hyperlinks_by_sheet: dict[str | int | None, dict[tuple[int, int], str]] = {}
+        if self.link_format != "none":
+            file_bytes.seek(0)
+            wb = openpyxl.load_workbook(file_bytes, data_only=True)
+            for sheet_key in sheet_to_dataframe:
+                if isinstance(sheet_key, int):
+                    ws = wb.worksheets[sheet_key]
+                elif sheet_key is None:
+                    ws = wb.active
+                else:
+                    ws = wb[sheet_key]
+                cell_links: dict[tuple[int, int], str] = {}
+                for row in ws.iter_rows():
+                    for cell in row:
+                        if cell.hyperlink and cell.hyperlink.target:
+                            # Convert to 0-based indices to match DataFrame positions
+                            cell_links[(cell.row - 1, cell.column - 1)] = cell.hyperlink.target
+                hyperlinks_by_sheet[sheet_key] = cell_links
+            wb.close()
+
+        updated_sheet_to_dataframe = {}
+        for key in sheet_to_dataframe:
+            df = sheet_to_dataframe[key]
+            # Row starts at 1 in Excel
+            df.index = df.index + 1
+            # Excel column names are Alphabet Characters
+            header = self._generate_excel_column_names(df.shape[1])
+            df.columns = header
+
+            # Apply hyperlinks to cell values
+            if key in hyperlinks_by_sheet:
+                for (row_idx, col_idx), url in hyperlinks_by_sheet[key].items():
+                    if row_idx < len(df) and col_idx < len(df.columns):
+                        cell_value = df.iat[row_idx, col_idx]
+                        text = str(cell_value) if pd.notna(cell_value) else ""
+                        if self.link_format == "markdown":
+                            df.iat[row_idx, col_idx] = f"[{text}]({url})"
+                        else:
+                            df.iat[row_idx, col_idx] = f"{text} ({url})"
+
+            updated_sheet_to_dataframe[key] = df
+
+        tables = []
+        metadata = []
+        for key, value in updated_sheet_to_dataframe.items():
+            if self.table_format == "csv":
+                resolved_kwargs = {"index": True, "header": True, "lineterminator": "\n", **self.table_format_kwargs}
+                tables.append(value.to_csv(**resolved_kwargs))
+            else:
+                resolved_kwargs = {
+                    "index": True,
+                    "headers": value.columns,
+                    "tablefmt": "pipe",
+                    **self.table_format_kwargs,
+                }
+                # to_markdown uses tabulate
+                tables.append(value.to_markdown(**resolved_kwargs))
+            # add sheet_name to metadata
+            metadata.append({"xlsx": {"sheet_name": key}})
+        return tables, metadata

haystack/components/embedders/__init__.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "azure_document_embedder": ["AzureOpenAIDocumentEmbedder"],
+    "azure_text_embedder": ["AzureOpenAITextEmbedder"],
+    "mock_document_embedder": ["MockDocumentEmbedder"],
+    "mock_text_embedder": ["MockTextEmbedder"],
+    "openai_document_embedder": ["OpenAIDocumentEmbedder"],
+    "openai_text_embedder": ["OpenAITextEmbedder"],
+}
+
+if TYPE_CHECKING:
+    from .azure_document_embedder import AzureOpenAIDocumentEmbedder as AzureOpenAIDocumentEmbedder
+    from .azure_text_embedder import AzureOpenAITextEmbedder as AzureOpenAITextEmbedder
+    from .mock_document_embedder import MockDocumentEmbedder as MockDocumentEmbedder
+    from .mock_text_embedder import MockTextEmbedder as MockTextEmbedder
+    from .openai_document_embedder import OpenAIDocumentEmbedder as OpenAIDocumentEmbedder
+    from .openai_text_embedder import OpenAITextEmbedder as OpenAITextEmbedder
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/embedders/azure_document_embedder.py

@@ -0,0 +1,250 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any
+
+from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.embedders import OpenAIDocumentEmbedder
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
+    """
+    Calculates document embeddings using OpenAI models deployed on Azure.
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack import Document
+    from haystack.components.embedders import AzureOpenAIDocumentEmbedder
+
+    doc = Document(content="I love pizza!")
+    document_embedder = AzureOpenAIDocumentEmbedder()
+
+    result = document_embedder.run([doc])
+    print(result['documents'][0].embedding)
+
+    # [0.017020374536514282, -0.023255806416273117, ...]
+    ```
+    """
+
+    def __init__(  # noqa: PLR0913 (too-many-arguments)
+        self,
+        azure_endpoint: str | None = None,
+        api_version: str | None = "2023-05-15",
+        azure_deployment: str = "text-embedding-ada-002",
+        dimensions: int | None = None,
+        api_key: Secret | None = Secret.from_env_var("AZURE_OPENAI_API_KEY", strict=False),
+        azure_ad_token: Secret | None = Secret.from_env_var("AZURE_OPENAI_AD_TOKEN", strict=False),
+        organization: str | None = None,
+        prefix: str = "",
+        suffix: str = "",
+        batch_size: int = 32,
+        progress_bar: bool = True,
+        meta_fields_to_embed: list[str] | None = None,
+        embedding_separator: str = "\n",
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        *,
+        default_headers: dict[str, str] | None = None,
+        azure_ad_token_provider: AzureADTokenProvider | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        Creates an AzureOpenAIDocumentEmbedder component.
+
+        :param azure_endpoint:
+            The endpoint of the model deployed on Azure.
+        :param api_version:
+            The version of the API to use.
+        :param azure_deployment:
+            The name of the model deployed on Azure. The default model is text-embedding-ada-002.
+        :param dimensions:
+            The number of dimensions of the resulting embeddings. Only supported in text-embedding-3
+            and later models.
+        :param api_key:
+            The Azure OpenAI API key.
+            You can set it with an environment variable `AZURE_OPENAI_API_KEY`, or pass with this
+            parameter during initialization.
+        :param azure_ad_token:
+            Microsoft Entra ID token, see Microsoft's
+            [Entra ID](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id)
+            documentation for more information. You can set it with an environment variable
+            `AZURE_OPENAI_AD_TOKEN`, or pass with this parameter during initialization.
+            Previously called Azure Active Directory.
+        :param organization:
+            Your organization ID. See OpenAI's
+            [Setting Up Your Organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization)
+            for more information.
+        :param prefix:
+            A string to add at the beginning of each text.
+        :param suffix:
+            A string to add at the end of each text.
+        :param batch_size:
+            Number of documents to embed at once.
+        :param progress_bar:
+            If `True`, shows a progress bar when running.
+        :param meta_fields_to_embed:
+            List of metadata fields to embed along with the document text.
+        :param embedding_separator:
+            Separator used to concatenate the metadata fields to the document text.
+        :param timeout: The timeout for `AzureOpenAI` client calls, in seconds.
+            If not set, defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact AzureOpenAI after an internal error.
+            If not set, defaults to either the `OPENAI_MAX_RETRIES` environment variable or to 5 retries.
+        :param default_headers: Default headers to send to the AzureOpenAI client.
+        :param azure_ad_token_provider: A function that returns an Azure Active Directory token, will be invoked on
+            every request.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        :param raise_on_failure:
+            Whether to raise an exception if the embedding request fails. If `False`, the component will log the error
+            and continue processing the remaining documents. If `True`, it will raise an exception on failure.
+        """
+        # We intentionally do not call super().__init__ here because we only need to instantiate the client to interact
+        # with the API.
+
+        # if not provided as a parameter, azure_endpoint is read from the env var AZURE_OPENAI_ENDPOINT
+        azure_endpoint = azure_endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")
+        if not azure_endpoint:
+            raise ValueError("Please provide an Azure endpoint or set the environment variable AZURE_OPENAI_ENDPOINT.")
+
+        if api_key is None and azure_ad_token is None:
+            raise ValueError("Please provide an API key or an Azure Active Directory token.")
+
+        self.api_key = api_key  # type: ignore[assignment] # mypy does not understand that api_key can be None
+        self.azure_ad_token = azure_ad_token
+        self.api_version = api_version
+        self.azure_endpoint = azure_endpoint
+        self.azure_deployment = azure_deployment
+        self.model = azure_deployment
+        self.dimensions = dimensions
+        self.organization = organization
+        self.prefix = prefix
+        self.suffix = suffix
+        self.batch_size = batch_size
+        self.progress_bar = progress_bar
+        self.meta_fields_to_embed = meta_fields_to_embed or []
+        self.embedding_separator = embedding_separator
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.default_headers = default_headers or {}
+        self.azure_ad_token_provider = azure_ad_token_provider
+        self.http_client_kwargs = http_client_kwargs
+        self.raise_on_failure = raise_on_failure
+
+        self.client: AzureOpenAI | None = None
+        self.async_client: AsyncAzureOpenAI | None = None
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_version": self.api_version,
+            "azure_endpoint": self.azure_endpoint,
+            "azure_deployment": self.azure_deployment,
+            "azure_ad_token_provider": self.azure_ad_token_provider,
+            "api_key": self.api_key.resolve_value() if self.api_key is not None else None,
+            "azure_ad_token": self.azure_ad_token.resolve_value() if self.azure_ad_token is not None else None,
+            "organization": self.organization,
+            "timeout": timeout,
+            "max_retries": max_retries,
+            "default_headers": self.default_headers,
+        }
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous AzureOpenAI client.
+        """
+        if self.client is None:
+            self.client = AzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous AzureOpenAI client on the serving event loop.
+        """
+        if self.async_client is None:
+            self.async_client = AsyncAzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous AzureOpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous AzureOpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        azure_ad_token_provider_name = None
+        if self.azure_ad_token_provider:
+            azure_ad_token_provider_name = serialize_callable(self.azure_ad_token_provider)
+        return default_to_dict(
+            self,
+            azure_endpoint=self.azure_endpoint,
+            azure_deployment=self.azure_deployment,
+            dimensions=self.dimensions,
+            organization=self.organization,
+            api_version=self.api_version,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            batch_size=self.batch_size,
+            progress_bar=self.progress_bar,
+            meta_fields_to_embed=self.meta_fields_to_embed,
+            embedding_separator=self.embedding_separator,
+            api_key=self.api_key,
+            azure_ad_token=self.azure_ad_token,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            default_headers=self.default_headers,
+            azure_ad_token_provider=azure_ad_token_provider_name,
+            http_client_kwargs=self.http_client_kwargs,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIDocumentEmbedder":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        serialized_azure_ad_token_provider = data["init_parameters"].get("azure_ad_token_provider")
+        if serialized_azure_ad_token_provider:
+            data["init_parameters"]["azure_ad_token_provider"] = deserialize_callable(
+                serialized_azure_ad_token_provider
+            )
+        return default_from_dict(cls, data)

haystack/components/embedders/azure_text_embedder.py

@@ -0,0 +1,226 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any
+
+from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.embedders import OpenAITextEmbedder
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+
+@component
+class AzureOpenAITextEmbedder(OpenAITextEmbedder):
+    """
+    Embeds strings using OpenAI models deployed on Azure.
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.embedders import AzureOpenAITextEmbedder
+
+    text_to_embed = "I love pizza!"
+    text_embedder = AzureOpenAITextEmbedder()
+
+    print(text_embedder.run(text_to_embed))
+
+    # {'embedding': [0.017020374536514282, -0.023255806416273117, ...],
+    # 'meta': {'model': 'text-embedding-ada-002-v2',
+    #          'usage': {'prompt_tokens': 4, 'total_tokens': 4}}}
+    ```
+    """
+
+    def __init__(  # noqa: PLR0913
+        self,
+        azure_endpoint: str | None = None,
+        api_version: str | None = "2023-05-15",
+        azure_deployment: str = "text-embedding-ada-002",
+        dimensions: int | None = None,
+        api_key: Secret | None = Secret.from_env_var("AZURE_OPENAI_API_KEY", strict=False),
+        azure_ad_token: Secret | None = Secret.from_env_var("AZURE_OPENAI_AD_TOKEN", strict=False),
+        organization: str | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        prefix: str = "",
+        suffix: str = "",
+        *,
+        default_headers: dict[str, str] | None = None,
+        azure_ad_token_provider: AzureADTokenProvider | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Creates an AzureOpenAITextEmbedder component.
+
+        :param azure_endpoint:
+            The endpoint of the model deployed on Azure.
+        :param api_version:
+            The version of the API to use.
+        :param azure_deployment:
+            The name of the model deployed on Azure. The default model is text-embedding-ada-002.
+        :param dimensions:
+            The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3
+            and later models.
+        :param api_key:
+            The Azure OpenAI API key.
+            You can set it with an environment variable `AZURE_OPENAI_API_KEY`, or pass with this
+            parameter during initialization.
+        :param azure_ad_token:
+            Microsoft Entra ID token, see Microsoft's
+            [Entra ID](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id)
+            documentation for more information. You can set it with an environment variable
+            `AZURE_OPENAI_AD_TOKEN`, or pass with this parameter during initialization.
+            Previously called Azure Active Directory.
+        :param organization:
+            Your organization ID. See OpenAI's
+            [Setting Up Your Organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization)
+            for more information.
+        :param timeout: The timeout for `AzureOpenAI` client calls, in seconds.
+            If not set, defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact AzureOpenAI after an internal error.
+            If not set, defaults to either the `OPENAI_MAX_RETRIES` environment variable, or to 5 retries.
+        :param prefix:
+            A string to add at the beginning of each text.
+        :param suffix:
+            A string to add at the end of each text.
+        :param default_headers: Default headers to send to the AzureOpenAI client.
+        :param azure_ad_token_provider: A function that returns an Azure Active Directory token, will be invoked on
+            every request.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+
+        """
+        # We intentionally do not call super().__init__ here because we only need to instantiate the client to interact
+        # with the API.
+
+        # Why is this here?
+        # AzureOpenAI init is forcing us to use an init method that takes either base_url or azure_endpoint as not
+        # None init parameters. This way we accommodate the use case where env var AZURE_OPENAI_ENDPOINT is set instead
+        # of passing it as a parameter.
+        azure_endpoint = azure_endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")
+        if not azure_endpoint:
+            raise ValueError("Please provide an Azure endpoint or set the environment variable AZURE_OPENAI_ENDPOINT.")
+
+        if api_key is None and azure_ad_token is None:
+            raise ValueError("Please provide an API key or an Azure Active Directory token.")
+
+        self.api_key = api_key  # type: ignore[assignment] # mypy does not understand that api_key can be None
+        self.azure_ad_token = azure_ad_token
+        self.api_version = api_version
+        self.azure_endpoint = azure_endpoint
+        self.azure_deployment = azure_deployment
+        self.model = azure_deployment
+        self.dimensions = dimensions
+        self.organization = organization
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.prefix = prefix
+        self.suffix = suffix
+        self.default_headers = default_headers or {}
+        self.azure_ad_token_provider = azure_ad_token_provider
+        self.http_client_kwargs = http_client_kwargs
+
+        self.client: AzureOpenAI | None = None
+        self.async_client: AsyncAzureOpenAI | None = None
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_version": self.api_version,
+            "azure_endpoint": self.azure_endpoint,
+            "azure_deployment": self.azure_deployment,
+            "azure_ad_token_provider": self.azure_ad_token_provider,
+            "api_key": self.api_key.resolve_value() if self.api_key is not None else None,
+            "azure_ad_token": self.azure_ad_token.resolve_value() if self.azure_ad_token is not None else None,
+            "organization": self.organization,
+            "timeout": timeout,
+            "max_retries": max_retries,
+            "default_headers": self.default_headers,
+        }
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous Azure OpenAI client.
+        """
+        if self.client is None:
+            self.client = AzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous Azure OpenAI client on the serving event loop.
+        """
+        if self.async_client is None:
+            self.async_client = AsyncAzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous Azure OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous Azure OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        azure_ad_token_provider_name = None
+        if self.azure_ad_token_provider:
+            azure_ad_token_provider_name = serialize_callable(self.azure_ad_token_provider)
+        return default_to_dict(
+            self,
+            azure_endpoint=self.azure_endpoint,
+            azure_deployment=self.azure_deployment,
+            dimensions=self.dimensions,
+            organization=self.organization,
+            api_version=self.api_version,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            api_key=self.api_key,
+            azure_ad_token=self.azure_ad_token,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            default_headers=self.default_headers,
+            azure_ad_token_provider=azure_ad_token_provider_name,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAITextEmbedder":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        serialized_azure_ad_token_provider = data["init_parameters"].get("azure_ad_token_provider")
+        if serialized_azure_ad_token_provider:
+            data["init_parameters"]["azure_ad_token_provider"] = deserialize_callable(
+                serialized_azure_ad_token_provider
+            )
+        return default_from_dict(cls, data)

haystack/components/embedders/mock_document_embedder.py

@@ -0,0 +1,194 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.components.embedders.mock_utils import (
+    EmbeddingFn,
+    _coerce_embedding,
+    _deterministic_embedding,
+    _estimate_usage,
+)
+from haystack.utils import deserialize_callable, serialize_callable
+
+
+@component
+class MockDocumentEmbedder:
+    """
+    A Document Embedder that returns deterministic embeddings without calling any API.
+
+    It is a drop-in replacement for real Document Embedders (such as `OpenAIDocumentEmbedder`) in tests, smoke tests,
+    and quick prototypes. It implements the same interface (`run`, `run_async`, serialization) but never contacts an
+    external service, so it is fully deterministic and free to run.
+
+    The embedding is selected based on how the component is configured:
+
+    - **Deterministic (default)**: with no configuration, each document's embedding is derived from a hash of its
+      (prepared) text. The same text always yields the same embedding, and different texts yield different
+      embeddings, so the mock works in retrieval pipelines and is reproducible across runs and processes.
+    - **Fixed embedding**: pass an `embedding` vector. The same vector is assigned to every document.
+    - **Dynamic embedding**: pass an `embedding_fn` callable that receives the (prepared) text of a document and
+      returns the embedding. This is useful when the embedding should depend on the input in a custom way.
+
+    Like real Document Embedders, the metadata fields listed in `meta_fields_to_embed` are concatenated with the
+    document content before embedding, so the deterministic embedding reflects the embedded metadata.
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.embedders import MockDocumentEmbedder
+
+    embedder = MockDocumentEmbedder(dimension=8)
+    result = embedder.run([Document(content="I love pizza!")])
+    print(result["documents"][0].embedding)  # a deterministic list of 8 floats
+    ```
+    """
+
+    def __init__(
+        self,
+        embedding: list[float] | None = None,
+        *,
+        embedding_fn: EmbeddingFn | None = None,
+        dimension: int = 768,
+        model: str = "mock-model",
+        meta: dict[str, Any] | None = None,
+        prefix: str = "",
+        suffix: str = "",
+        meta_fields_to_embed: list[str] | None = None,
+        embedding_separator: str = "\n",
+        progress_bar: bool = False,
+    ) -> None:
+        """
+        Creates an instance of MockDocumentEmbedder.
+
+        :param embedding: An optional fixed embedding assigned to every document. Mutually exclusive with
+            `embedding_fn`. If neither is provided, a deterministic embedding is derived from each document's text.
+        :param embedding_fn: An optional callable that receives the prepared text of a document and returns the
+            embedding as a list of floats. Mutually exclusive with `embedding`. To support serialization, pass a
+            named function (lambdas and nested functions cannot be serialized).
+        :param dimension: The number of dimensions of the deterministic embedding. Ignored when `embedding` or
+            `embedding_fn` is provided, since their length is determined by the value or callable.
+        :param model: The model name reported in the metadata. Purely cosmetic; no model is loaded.
+        :param meta: Additional metadata merged into the output `meta`.
+        :param prefix: A string to add at the beginning of each text before embedding.
+        :param suffix: A string to add at the end of each text before embedding.
+        :param meta_fields_to_embed: List of metadata fields to embed along with the document text.
+        :param embedding_separator: Separator used to concatenate the metadata fields to the document text.
+        :param progress_bar: Accepted for interface compatibility with real Document Embedders and ignored.
+        :raises ValueError: If both `embedding` and `embedding_fn` are provided, if `dimension` is not positive, or
+            if `embedding` is an empty list.
+        :raises TypeError: If `embedding` is not a sequence of numbers.
+        """
+        if embedding is not None and embedding_fn is not None:
+            raise ValueError("Pass either 'embedding' or 'embedding_fn', not both.")
+        if dimension <= 0:
+            raise ValueError("'dimension' must be a positive integer.")
+
+        self.embedding = _coerce_embedding(embedding, name="'embedding'") if embedding is not None else None
+        self.embedding_fn = embedding_fn
+        self.dimension = dimension
+        self.model = model
+        self.meta = meta or {}
+        self.prefix = prefix
+        self.suffix = suffix
+        self.meta_fields_to_embed = meta_fields_to_embed or []
+        self.embedding_separator = embedding_separator
+        self.progress_bar = progress_bar
+        self._is_warmed_up = False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        embedding_fn = serialize_callable(self.embedding_fn) if self.embedding_fn is not None else None
+        return default_to_dict(
+            self,
+            embedding=self.embedding,
+            embedding_fn=embedding_fn,
+            dimension=self.dimension,
+            model=self.model,
+            meta=self.meta,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            meta_fields_to_embed=self.meta_fields_to_embed,
+            embedding_separator=self.embedding_separator,
+            progress_bar=self.progress_bar,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> MockDocumentEmbedder:
+        """Deserialize the component from a dictionary."""
+        init_params = data.get("init_parameters", {})
+        embedding_fn = init_params.get("embedding_fn")
+        if embedding_fn:
+            init_params["embedding_fn"] = deserialize_callable(embedding_fn)
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """No-op warm up, provided for interface compatibility with real Embedders."""
+        self._is_warmed_up = True
+
+    def _prepare_text_to_embed(self, document: Document) -> str:
+        """Concatenate the document content with the metadata fields to embed, mirroring real Document Embedders."""
+        meta_values_to_embed = [
+            str(document.meta[key])
+            for key in self.meta_fields_to_embed
+            if key in document.meta and document.meta[key] is not None
+        ]
+        return (
+            self.prefix + self.embedding_separator.join([*meta_values_to_embed, document.content or ""]) + self.suffix
+        )
+
+    def _embed(self, text: str) -> list[float]:
+        """Produce the embedding for the prepared text according to the configured mode."""
+        if self.embedding_fn is not None:
+            return _coerce_embedding(self.embedding_fn(text), name="the return value of 'embedding_fn'")
+        if self.embedding is not None:
+            return list(self.embedding)
+        return _deterministic_embedding(text, self.dimension)
+
+    @component.output_types(documents=list[Document], meta=dict[str, Any])
+    def run(self, documents: list[Document]) -> dict[str, Any]:
+        """
+        Return the input documents with deterministic embeddings added, without calling any API.
+
+        :param documents: A list of documents to embed.
+        :returns: A dictionary with the following keys:
+            - `documents`: A list of documents with embeddings.
+            - `meta`: Metadata about the (mock) model.
+        :raises TypeError: If `documents` is not a list of `Document` objects.
+        """
+        self.warm_up()
+
+        if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
+            raise TypeError(
+                "MockDocumentEmbedder expects a list of Documents as input. "
+                "In case you want to embed a string, please use the MockTextEmbedder."
+            )
+
+        texts_to_embed = [self._prepare_text_to_embed(document) for document in documents]
+        new_documents = [
+            replace(document, embedding=self._embed(text))
+            for document, text in zip(documents, texts_to_embed, strict=True)
+        ]
+
+        meta: dict[str, Any] = {"model": self.model, "usage": _estimate_usage(texts_to_embed)}
+        meta.update(self.meta)
+        return {"documents": new_documents, "meta": meta}
+
+    @component.output_types(documents=list[Document], meta=dict[str, Any])
+    async def run_async(self, documents: list[Document]) -> dict[str, Any]:
+        """
+        Asynchronously return the input documents with deterministic embeddings added, without calling any API.
+
+        :param documents: A list of documents to embed.
+        :returns: A dictionary with the following keys:
+            - `documents`: A list of documents with embeddings.
+            - `meta`: Metadata about the (mock) model.
+        :raises TypeError: If `documents` is not a list of `Document` objects.
+        """
+        return self.run(documents=documents)

haystack/components/embedders/mock_text_embedder.py

@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.embedders.mock_utils import (
+    EmbeddingFn,
+    _coerce_embedding,
+    _deterministic_embedding,
+    _estimate_usage,
+)
+from haystack.utils import deserialize_callable, serialize_callable
+
+
+@component
+class MockTextEmbedder:
+    """
+    A Text Embedder that returns deterministic embeddings without calling any API.
+
+    It is a drop-in replacement for real Text Embedders (such as `OpenAITextEmbedder`) in tests, smoke tests, and
+    quick prototypes. It implements the same interface (`run`, `run_async`, serialization) but never contacts an
+    external service, so it is fully deterministic and free to run.
+
+    The embedding is selected based on how the component is configured:
+
+    - **Deterministic (default)**: with no configuration, the embedding is derived from a hash of the input text.
+      The same text always yields the same embedding, and different texts yield different embeddings, so the mock
+      works in retrieval pipelines and is reproducible across runs and processes.
+    - **Fixed embedding**: pass an `embedding` vector. The same vector is returned for every input.
+    - **Dynamic embedding**: pass an `embedding_fn` callable that receives the (prepared) text and returns the
+      embedding. This is useful when the embedding should depend on the input in a custom way.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.embedders import MockTextEmbedder
+
+    embedder = MockTextEmbedder(dimension=8)
+    result = embedder.run("I love pizza!")
+    print(result["embedding"])  # a deterministic list of 8 floats
+    ```
+    """
+
+    def __init__(
+        self,
+        embedding: list[float] | None = None,
+        *,
+        embedding_fn: EmbeddingFn | None = None,
+        dimension: int = 768,
+        model: str = "mock-model",
+        meta: dict[str, Any] | None = None,
+        prefix: str = "",
+        suffix: str = "",
+    ) -> None:
+        """
+        Creates an instance of MockTextEmbedder.
+
+        :param embedding: An optional fixed embedding returned for every input. Mutually exclusive with
+            `embedding_fn`. If neither is provided, a deterministic embedding is derived from the input text.
+        :param embedding_fn: An optional callable that receives the prepared text (after `prefix`/`suffix` are
+            applied) and returns the embedding as a list of floats. Mutually exclusive with `embedding`. To support
+            serialization, pass a named function (lambdas and nested functions cannot be serialized).
+        :param dimension: The number of dimensions of the deterministic embedding. Ignored when `embedding` or
+            `embedding_fn` is provided, since their length is determined by the value or callable.
+        :param model: The model name reported in the metadata. Purely cosmetic; no model is loaded.
+        :param meta: Additional metadata merged into the output `meta`.
+        :param prefix: A string to add at the beginning of the text before embedding.
+        :param suffix: A string to add at the end of the text before embedding.
+        :raises ValueError: If both `embedding` and `embedding_fn` are provided, if `dimension` is not positive, or
+            if `embedding` is an empty list.
+        :raises TypeError: If `embedding` is not a sequence of numbers.
+        """
+        if embedding is not None and embedding_fn is not None:
+            raise ValueError("Pass either 'embedding' or 'embedding_fn', not both.")
+        if dimension <= 0:
+            raise ValueError("'dimension' must be a positive integer.")
+
+        self.embedding = _coerce_embedding(embedding, name="'embedding'") if embedding is not None else None
+        self.embedding_fn = embedding_fn
+        self.dimension = dimension
+        self.model = model
+        self.meta = meta or {}
+        self.prefix = prefix
+        self.suffix = suffix
+        self._is_warmed_up = False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        embedding_fn = serialize_callable(self.embedding_fn) if self.embedding_fn is not None else None
+        return default_to_dict(
+            self,
+            embedding=self.embedding,
+            embedding_fn=embedding_fn,
+            dimension=self.dimension,
+            model=self.model,
+            meta=self.meta,
+            prefix=self.prefix,
+            suffix=self.suffix,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> MockTextEmbedder:
+        """Deserialize the component from a dictionary."""
+        init_params = data.get("init_parameters", {})
+        embedding_fn = init_params.get("embedding_fn")
+        if embedding_fn:
+            init_params["embedding_fn"] = deserialize_callable(embedding_fn)
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """No-op warm up, provided for interface compatibility with real Embedders."""
+        self._is_warmed_up = True
+
+    def _embed(self, text: str) -> list[float]:
+        """Produce the embedding for the prepared text according to the configured mode."""
+        if self.embedding_fn is not None:
+            return _coerce_embedding(self.embedding_fn(text), name="the return value of 'embedding_fn'")
+        if self.embedding is not None:
+            return list(self.embedding)
+        return _deterministic_embedding(text, self.dimension)
+
+    @component.output_types(embedding=list[float], meta=dict[str, Any])
+    def run(self, text: str) -> dict[str, Any]:
+        """
+        Return a deterministic embedding for the input text without calling any API.
+
+        :param text: The text to embed.
+        :returns: A dictionary with the following keys:
+            - `embedding`: The embedding of the input text.
+            - `meta`: Metadata about the (mock) model.
+        :raises TypeError: If `text` is not a string.
+        """
+        self.warm_up()
+
+        if not isinstance(text, str):
+            raise TypeError(
+                "MockTextEmbedder expects a string as an input. "
+                "In case you want to embed a list of Documents, please use the MockDocumentEmbedder."
+            )
+
+        text_to_embed = self.prefix + text + self.suffix
+        meta: dict[str, Any] = {"model": self.model, "usage": _estimate_usage([text_to_embed])}
+        meta.update(self.meta)
+        return {"embedding": self._embed(text_to_embed), "meta": meta}
+
+    @component.output_types(embedding=list[float], meta=dict[str, Any])
+    async def run_async(self, text: str) -> dict[str, Any]:
+        """
+        Asynchronously return a deterministic embedding for the input text without calling any API.
+
+        :param text: The text to embed.
+        :returns: A dictionary with the following keys:
+            - `embedding`: The embedding of the input text.
+            - `meta`: Metadata about the (mock) model.
+        :raises TypeError: If `text` is not a string.
+        """
+        return self.run(text=text)

haystack/components/embedders/mock_utils.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import hashlib
+import math
+import random
+from collections.abc import Callable
+
+# A callable that derives an embedding from the (prepared) text to embed. It receives the text and returns the
+# embedding as a list of floats.
+EmbeddingFn = Callable[[str], list[float]]
+
+
+def _l2_normalize(vector: list[float]) -> list[float]:
+    """Return the L2-normalized vector, so that mock embeddings behave like real (unit-length) ones."""
+    norm = math.sqrt(sum(value * value for value in vector))
+    if norm == 0.0:
+        return vector
+    return [value / norm for value in vector]
+
+
+def _deterministic_embedding(text: str, dimension: int) -> list[float]:
+    """
+    Generate a deterministic, unit-length embedding from the given text.
+
+    The same text always yields the same embedding, and different texts yield different embeddings, which makes mock
+    embeddings usable in retrieval pipelines and reproducible across runs and processes. The seed is derived from a
+    SHA-256 digest of the text (not the process-salted built-in `hash`) to guarantee cross-process stability.
+
+    :param text: The text to embed.
+    :param dimension: The number of dimensions of the resulting embedding.
+    :returns: A deterministic, L2-normalized embedding of length `dimension`.
+    """
+    digest = hashlib.sha256(text.encode("utf-8")).digest()
+    seed = int.from_bytes(digest[:8], "big")
+    rng = random.Random(seed)
+    vector = [rng.uniform(-1.0, 1.0) for _ in range(dimension)]
+    return _l2_normalize(vector)
+
+
+def _coerce_embedding(value: object, *, name: str) -> list[float]:
+    """
+    Validate that `value` is a non-empty sequence of numbers and coerce it into a list of floats.
+
+    :param value: The value to validate, e.g. a user-provided fixed embedding or the output of an `embedding_fn`.
+    :param name: How to refer to `value` in error messages, e.g. ``"'embedding'"``.
+    """
+    if not isinstance(value, (list, tuple)) or not all(isinstance(item, (int, float)) for item in value):
+        raise TypeError(f"{name} must be a sequence of numbers, got {type(value)}.")
+    if len(value) == 0:
+        raise ValueError(f"{name} must not be empty.")
+    return [float(item) for item in value]
+
+
+def _estimate_usage(texts: list[str]) -> dict[str, int]:
+    """
+    Roughly estimate token usage as whitespace-separated word counts.
+
+    This is an approximation (not real tokenization) intended to give downstream code realistic-looking metadata.
+    """
+    prompt_tokens = sum(len(text.split()) for text in texts)
+    return {"prompt_tokens": prompt_tokens, "total_tokens": prompt_tokens}

haystack/components/embedders/openai_document_embedder.py

@@ -0,0 +1,390 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from dataclasses import replace
+from typing import Any
+
+from more_itertools import batched
+from openai import APIError, AsyncOpenAI, OpenAI
+from tqdm import tqdm
+from tqdm.asyncio import tqdm as async_tqdm
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.utils import Secret
+from haystack.utils.http_client import init_http_client
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class OpenAIDocumentEmbedder:
+    """
+    Computes document embeddings using OpenAI models.
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack import Document
+    from haystack.components.embedders import OpenAIDocumentEmbedder
+
+    doc = Document(content="I love pizza!")
+    document_embedder = OpenAIDocumentEmbedder()
+    result = document_embedder.run([doc])
+
+    print(result['documents'][0].embedding)
+
+    # [0.017020374536514282, -0.023255806416273117, ...]
+    ```
+    """
+
+    def __init__(  # noqa: PLR0913 (too-many-arguments)
+        self,
+        api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
+        model: str = "text-embedding-ada-002",
+        dimensions: int | None = None,
+        api_base_url: str | None = None,
+        organization: str | None = None,
+        prefix: str = "",
+        suffix: str = "",
+        batch_size: int = 32,
+        progress_bar: bool = True,
+        meta_fields_to_embed: list[str] | None = None,
+        embedding_separator: str = "\n",
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+        *,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        Creates an OpenAIDocumentEmbedder component.
+
+        Before initializing the component, you can set the 'OPENAI_TIMEOUT' and 'OPENAI_MAX_RETRIES'
+        environment variables to override the `timeout` and `max_retries` parameters respectively
+        in the OpenAI client.
+
+        :param api_key:
+            The OpenAI API key.
+            You can set it with an environment variable `OPENAI_API_KEY`, or pass with this parameter
+            during initialization.
+        :param model:
+            The name of the model to use for calculating embeddings.
+            The default model is `text-embedding-ada-002`.
+        :param dimensions:
+            The number of dimensions of the resulting embeddings. Only `text-embedding-3` and
+            later models support this parameter.
+        :param api_base_url:
+            Overrides the default base URL for all HTTP requests.
+        :param organization:
+            Your OpenAI organization ID. See OpenAI's
+            [Setting Up Your Organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization)
+            for more information.
+        :param prefix:
+            A string to add at the beginning of each text.
+        :param suffix:
+            A string to add at the end of each text.
+        :param batch_size:
+            Number of documents to embed at once.
+        :param progress_bar:
+            If `True`, shows a progress bar when running.
+        :param meta_fields_to_embed:
+            List of metadata fields to embed along with the document text.
+        :param embedding_separator:
+            Separator used to concatenate the metadata fields to the document text.
+        :param timeout:
+            Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries:
+            Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or 5 retries.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        :param raise_on_failure:
+            Whether to raise an exception if the embedding request fails. If `False`, the component will log the error
+            and continue processing the remaining documents. If `True`, it will raise an exception on failure.
+        """
+        self.api_key = api_key
+        self.model = model
+        self.dimensions = dimensions
+        self.api_base_url = api_base_url
+        self.organization = organization
+        self.prefix = prefix
+        self.suffix = suffix
+        self.batch_size = batch_size
+        self.progress_bar = progress_bar
+        self.meta_fields_to_embed = meta_fields_to_embed or []
+        self.embedding_separator = embedding_separator
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.http_client_kwargs = http_client_kwargs
+        self.raise_on_failure = raise_on_failure
+
+        self.client: OpenAI | None = None
+        self.async_client: AsyncOpenAI | None = None
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_key": self.api_key.resolve_value(),
+            "organization": self.organization,
+            "base_url": self.api_base_url,
+            "timeout": timeout,
+            "max_retries": max_retries,
+        }
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous OpenAI client.
+        """
+        if self.client is None:
+            self.client = OpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous OpenAI client on the serving event loop.
+        """
+        if self.async_client is None:
+            self.async_client = AsyncOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            api_key=self.api_key,
+            model=self.model,
+            dimensions=self.dimensions,
+            api_base_url=self.api_base_url,
+            organization=self.organization,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            batch_size=self.batch_size,
+            progress_bar=self.progress_bar,
+            meta_fields_to_embed=self.meta_fields_to_embed,
+            embedding_separator=self.embedding_separator,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            http_client_kwargs=self.http_client_kwargs,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OpenAIDocumentEmbedder":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    def _prepare_texts_to_embed(self, documents: list[Document]) -> dict[str, str]:
+        """
+        Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
+        """
+        texts_to_embed = {}
+        for doc in documents:
+            meta_values_to_embed = [
+                str(doc.meta[key]) for key in self.meta_fields_to_embed if key in doc.meta and doc.meta[key] is not None
+            ]
+
+            texts_to_embed[doc.id] = (
+                self.prefix + self.embedding_separator.join(meta_values_to_embed + [doc.content or ""]) + self.suffix
+            )
+
+        return texts_to_embed
+
+    def _embed_batch(
+        self, texts_to_embed: dict[str, str], batch_size: int
+    ) -> tuple[dict[str, list[float]], dict[str, Any]]:
+        """
+        Embed a list of texts in batches.
+        """
+
+        doc_ids_to_embeddings: dict[str, list[float]] = {}
+        meta: dict[str, Any] = {}
+        for batch in tqdm(
+            batched(texts_to_embed.items(), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
+        ):
+            args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch], "encoding_format": "float"}
+
+            if self.dimensions is not None:
+                args["dimensions"] = self.dimensions
+
+            try:
+                # this method is invoked after warm_up, so client is not None
+                assert self.client is not None
+                response = self.client.embeddings.create(**args)
+            except APIError as exc:
+                ids = ", ".join(b[0] for b in batch)
+                msg = "Failed embedding of documents {ids} caused by {exc}"
+                logger.exception(msg, ids=ids, exc=exc)
+                if self.raise_on_failure:
+                    raise exc
+                continue
+
+            embeddings = [el.embedding for el in response.data]
+            doc_ids_to_embeddings.update(dict(zip((b[0] for b in batch), embeddings, strict=True)))
+
+            if "model" not in meta:
+                meta["model"] = response.model
+            if "usage" not in meta:
+                meta["usage"] = dict(response.usage)
+            else:
+                meta["usage"]["prompt_tokens"] += response.usage.prompt_tokens
+                meta["usage"]["total_tokens"] += response.usage.total_tokens
+
+        return doc_ids_to_embeddings, meta
+
+    async def _embed_batch_async(
+        self, texts_to_embed: dict[str, str], batch_size: int
+    ) -> tuple[dict[str, list[float]], dict[str, Any]]:
+        """
+        Embed a list of texts in batches asynchronously.
+        """
+
+        doc_ids_to_embeddings: dict[str, list[float]] = {}
+        meta: dict[str, Any] = {}
+
+        batches = list(batched(texts_to_embed.items(), batch_size))
+        if self.progress_bar:
+            batches = async_tqdm(batches, desc="Calculating embeddings")
+
+        for batch in batches:
+            args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch]}
+
+            if self.dimensions is not None:
+                args["dimensions"] = self.dimensions
+
+            try:
+                # this method is invoked after warm_up_async, so async_client is not None
+                assert self.async_client is not None
+                response = await self.async_client.embeddings.create(**args)
+            except APIError as exc:
+                ids = ", ".join(b[0] for b in batch)
+                msg = "Failed embedding of documents {ids} caused by {exc}"
+                logger.exception(msg, ids=ids, exc=exc)
+                if self.raise_on_failure:
+                    raise exc
+                continue
+
+            embeddings = [el.embedding for el in response.data]
+            doc_ids_to_embeddings.update(dict(zip((b[0] for b in batch), embeddings, strict=True)))
+
+            if "model" not in meta:
+                meta["model"] = response.model
+            if "usage" not in meta:
+                meta["usage"] = dict(response.usage)
+            else:
+                meta["usage"]["prompt_tokens"] += response.usage.prompt_tokens
+                meta["usage"]["total_tokens"] += response.usage.total_tokens
+
+        return doc_ids_to_embeddings, meta
+
+    @component.output_types(documents=list[Document], meta=dict[str, Any])
+    def run(self, documents: list[Document]) -> dict[str, Any]:
+        """
+        Embeds a list of documents.
+
+        :param documents:
+            A list of documents to embed.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: A list of documents with embeddings.
+            - `meta`: Information about the usage of the model.
+        """
+        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
+            raise TypeError(
+                "OpenAIDocumentEmbedder expects a list of Documents as input."
+                "In case you want to embed a string, please use the OpenAITextEmbedder."
+            )
+
+        self.warm_up()
+
+        texts_to_embed = self._prepare_texts_to_embed(documents=documents)
+
+        doc_ids_to_embeddings, meta = self._embed_batch(texts_to_embed=texts_to_embed, batch_size=self.batch_size)
+
+        new_documents = []
+        for doc in documents:
+            if doc.id in doc_ids_to_embeddings:
+                new_documents.append(replace(doc, embedding=doc_ids_to_embeddings[doc.id]))
+            else:
+                new_documents.append(replace(doc))
+
+        return {"documents": new_documents, "meta": meta}
+
+    @component.output_types(documents=list[Document], meta=dict[str, Any])
+    async def run_async(self, documents: list[Document]) -> dict[str, Any]:
+        """
+        Embeds a list of documents asynchronously.
+
+        :param documents:
+            A list of documents to embed.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: A list of documents with embeddings.
+            - `meta`: Information about the usage of the model.
+        """
+        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
+            raise TypeError(
+                "OpenAIDocumentEmbedder expects a list of Documents as input. "
+                "In case you want to embed a string, please use the OpenAITextEmbedder."
+            )
+
+        await self.warm_up_async()
+
+        texts_to_embed = self._prepare_texts_to_embed(documents=documents)
+
+        doc_ids_to_embeddings, meta = await self._embed_batch_async(
+            texts_to_embed=texts_to_embed, batch_size=self.batch_size
+        )
+
+        new_documents = []
+        for doc in documents:
+            if doc.id in doc_ids_to_embeddings:
+                new_documents.append(replace(doc, embedding=doc_ids_to_embeddings[doc.id]))
+            else:
+                new_documents.append(replace(doc))
+
+        return {"documents": new_documents, "meta": meta}

haystack/components/embedders/openai_text_embedder.py

@@ -0,0 +1,245 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any
+
+from openai import AsyncOpenAI, OpenAI
+from openai.types import CreateEmbeddingResponse
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.utils import Secret
+from haystack.utils.http_client import init_http_client
+
+
+@component
+class OpenAITextEmbedder:
+    """
+    Embeds strings using OpenAI models.
+
+    You can use it to embed user query and send it to an embedding Retriever.
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.embedders import OpenAITextEmbedder
+
+    text_to_embed = "I love pizza!"
+    text_embedder = OpenAITextEmbedder()
+
+    print(text_embedder.run(text_to_embed))
+
+    # {'embedding': [0.017020374536514282, -0.023255806416273117, ...],
+    # 'meta': {'model': 'text-embedding-ada-002-v2',
+    #          'usage': {'prompt_tokens': 4, 'total_tokens': 4}}}
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
+        model: str = "text-embedding-ada-002",
+        dimensions: int | None = None,
+        api_base_url: str | None = None,
+        organization: str | None = None,
+        prefix: str = "",
+        suffix: str = "",
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Creates an OpenAITextEmbedder component.
+
+        Before initializing the component, you can set the 'OPENAI_TIMEOUT' and 'OPENAI_MAX_RETRIES'
+        environment variables to override the `timeout` and `max_retries` parameters respectively
+        in the OpenAI client.
+
+        :param api_key:
+            The OpenAI API key.
+            You can set it with an environment variable `OPENAI_API_KEY`, or pass with this parameter
+            during initialization.
+        :param model:
+            The name of the model to use for calculating embeddings.
+            The default model is `text-embedding-ada-002`.
+        :param dimensions:
+            The number of dimensions of the resulting embeddings. Only `text-embedding-3` and
+            later models support this parameter.
+        :param api_base_url:
+            Overrides default base URL for all HTTP requests.
+        :param organization:
+            Your organization ID. See OpenAI's
+            [production best practices](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization)
+            for more information.
+        :param prefix:
+            A string to add at the beginning of each text to embed.
+        :param suffix:
+            A string to add at the end of each text to embed.
+        :param timeout:
+            Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries:
+            Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        self.model = model
+        self.dimensions = dimensions
+        self.api_base_url = api_base_url
+        self.organization = organization
+        self.prefix = prefix
+        self.suffix = suffix
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.http_client_kwargs = http_client_kwargs
+
+        self.client: OpenAI | None = None
+        self.async_client: AsyncOpenAI | None = None
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_key": self.api_key.resolve_value(),
+            "organization": self.organization,
+            "base_url": self.api_base_url,
+            "timeout": timeout,
+            "max_retries": max_retries,
+        }
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous OpenAI client.
+        """
+        if self.client is None:
+            self.client = OpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous OpenAI client on the serving event loop.
+        """
+        if self.async_client is None:
+            self.async_client = AsyncOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            api_key=self.api_key,
+            model=self.model,
+            dimensions=self.dimensions,
+            api_base_url=self.api_base_url,
+            organization=self.organization,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OpenAITextEmbedder":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        return default_from_dict(cls, data)
+
+    def _prepare_input(self, text: str) -> dict[str, Any]:
+        if not isinstance(text, str):
+            raise TypeError(
+                "OpenAITextEmbedder expects a string as an input."
+                "In case you want to embed a list of Documents, please use the OpenAIDocumentEmbedder."
+            )
+
+        text_to_embed = self.prefix + text + self.suffix
+
+        kwargs: dict[str, Any] = {"model": self.model, "input": text_to_embed, "encoding_format": "float"}
+        if self.dimensions is not None:
+            kwargs["dimensions"] = self.dimensions
+        return kwargs
+
+    def _prepare_output(self, result: CreateEmbeddingResponse) -> dict[str, Any]:
+        return {"embedding": result.data[0].embedding, "meta": {"model": result.model, "usage": dict(result.usage)}}
+
+    @component.output_types(embedding=list[float], meta=dict[str, Any])
+    def run(self, text: str) -> dict[str, Any]:
+        """
+        Embeds a single string.
+
+        :param text:
+            Text to embed.
+
+        :returns:
+            A dictionary with the following keys:
+            - `embedding`: The embedding of the input text.
+            - `meta`: Information about the usage of the model.
+        """
+        self.warm_up()
+        create_kwargs = self._prepare_input(text=text)
+        assert self.client is not None  # mypy: client is built by warm_up above
+        response = self.client.embeddings.create(**create_kwargs)
+        return self._prepare_output(result=response)
+
+    @component.output_types(embedding=list[float], meta=dict[str, Any])
+    async def run_async(self, text: str) -> dict[str, Any]:
+        """
+        Asynchronously embed a single string.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in async code.
+
+        :param text:
+            Text to embed.
+
+        :returns:
+            A dictionary with the following keys:
+            - `embedding`: The embedding of the input text.
+            - `meta`: Information about the usage of the model.
+        """
+        await self.warm_up_async()
+        create_kwargs = self._prepare_input(text=text)
+        assert self.async_client is not None  # mypy: async_client is built by warm_up_async above
+        response = await self.async_client.embeddings.create(**create_kwargs)
+        return self._prepare_output(result=response)

haystack/components/embedders/types/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .protocol import DocumentEmbedder, TextEmbedder
+
+__all__ = ["DocumentEmbedder", "TextEmbedder"]

haystack/components/embedders/types/protocol.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Protocol
+
+from haystack import Document
+
+
+class TextEmbedder(Protocol):
+    """
+    Protocol for Text Embedders.
+    """
+
+    def run(self, text: str) -> dict[str, Any]:
+        """
+        Generate embeddings for the input text.
+
+        Implementing classes may accept additional optional parameters in their run method.
+        For example: `def run (self, text: str, param_a="default", param_b="another_default")`.
+
+        :param text:
+            The input text to be embedded.
+        :returns:
+            A dictionary containing the keys:
+                - 'embedding', which is expected to be a list[float] representing the embedding.
+                - any optional keys such as 'metadata'.
+        """
+        ...
+
+
+class DocumentEmbedder(Protocol):
+    """
+    Protocol for Document Embedders.
+    """
+
+    def run(self, documents: list[Document]) -> dict[str, Any]:
+        """
+        Generate embeddings for the input documents.
+
+        Implementing classes may accept additional optional parameters in their run method.
+        For example: `def run (self, documents: list[Document], param_a="default", param_b="another_default")`.
+
+        :param documents:
+            The input documents to be embedded.
+        :returns:
+            A dictionary containing the keys:
+                - 'documents', which is expected to be a list[Document] with embeddings added to each document.
+                - any optional keys such as 'metadata'.
+        """
+        ...

haystack/components/evaluators/__init__.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "answer_exact_match": ["AnswerExactMatchEvaluator"],
+    "context_relevance": ["ContextRelevanceEvaluator"],
+    "document_map": ["DocumentMAPEvaluator"],
+    "document_mrr": ["DocumentMRREvaluator"],
+    "document_ndcg": ["DocumentNDCGEvaluator"],
+    "document_recall": ["DocumentRecallEvaluator"],
+    "faithfulness": ["FaithfulnessEvaluator"],
+    "llm_evaluator": ["LLMEvaluator"],
+    "sas_evaluator": ["SASEvaluator"],
+}
+
+if TYPE_CHECKING:
+    from .answer_exact_match import AnswerExactMatchEvaluator as AnswerExactMatchEvaluator
+    from .context_relevance import ContextRelevanceEvaluator as ContextRelevanceEvaluator
+    from .document_map import DocumentMAPEvaluator as DocumentMAPEvaluator
+    from .document_mrr import DocumentMRREvaluator as DocumentMRREvaluator
+    from .document_ndcg import DocumentNDCGEvaluator as DocumentNDCGEvaluator
+    from .document_recall import DocumentRecallEvaluator as DocumentRecallEvaluator
+    from .faithfulness import FaithfulnessEvaluator as FaithfulnessEvaluator
+    from .llm_evaluator import LLMEvaluator as LLMEvaluator
+    from .sas_evaluator import SASEvaluator as SASEvaluator
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/evaluators/answer_exact_match.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack.core.component import component
+
+
+@component
+class AnswerExactMatchEvaluator:
+    """
+    An answer exact match evaluator class.
+
+    The evaluator that checks if the predicted answers matches any of the ground truth answers exactly.
+    The result is a number from 0.0 to 1.0, it represents the proportion of predicted answers
+    that matched one of the ground truth answers.
+    There can be multiple ground truth answers and multiple predicted answers as input.
+
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import AnswerExactMatchEvaluator
+
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(
+        ground_truth_answers=["Berlin", "Paris"],
+        predicted_answers=["Berlin", "Lyon"],
+    )
+
+    print(result["individual_scores"])
+    # [1, 0]
+    print(result["score"])
+    # 0.5
+    ```
+    """
+
+    @component.output_types(individual_scores=list[int], score=float)
+    def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
+        """
+        Run the AnswerExactMatchEvaluator on the given inputs.
+
+        The `ground_truth_answers` and `retrieved_answers` must have the same length.
+
+        :param ground_truth_answers:
+            A list of expected answers.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+            - `individual_scores` - A list of 0s and 1s, where 1 means that the predicted answer matched one of the
+                ground truth.
+            - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted
+                         answer matched one of the ground truth answers.
+        """
+        if not len(ground_truth_answers) == len(predicted_answers):
+            raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.")
+
+        matches = []
+        for truth, extracted in zip(ground_truth_answers, predicted_answers, strict=True):
+            if truth == extracted:
+                matches.append(1)
+            else:
+                matches.append(0)
+
+        # The proportion of questions where any predicted answer matched one of the ground truth answers
+        average = sum(matches) / len(predicted_answers)
+
+        return {"individual_scores": matches, "score": average}

haystack/components/evaluators/context_relevance.py

@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+from statistics import mean
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.evaluators.llm_evaluator import LLMEvaluator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.utils import deserialize_chatgenerator_inplace
+
+logger = logging.getLogger(__name__)
+
+# Private global variable for default examples to include in the prompt if the user does not provide any examples
+_DEFAULT_EXAMPLES = [
+    {
+        "inputs": {
+            "questions": "What is the capital of Germany?",
+            "contexts": ["Berlin is the capital of Germany. Berlin and was founded in 1244."],
+        },
+        "outputs": {"relevant_statements": ["Berlin is the capital of Germany."]},
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of France?",
+            "contexts": [
+                "Berlin is the capital of Germany and was founded in 1244.",
+                "Europe is a continent with 44 countries.",
+                "Madrid is the capital of Spain.",
+            ],
+        },
+        "outputs": {"relevant_statements": []},
+    },
+    {
+        "inputs": {"questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."]},
+        "outputs": {"relevant_statements": ["Rome is the capital of Italy."]},
+    },
+]
+
+
+@component
+class ContextRelevanceEvaluator(LLMEvaluator):
+    """
+    Evaluator that checks if a provided context is relevant to the question.
+
+    An LLM breaks up a context into multiple statements and checks whether each statement
+    is relevant for answering a question.
+    The score for each context is either binary score of 1 or 0, where 1 indicates that the context is relevant
+    to the question and 0 indicates that the context is not relevant.
+    The evaluator also provides the relevant statements from the context and an average score over all the provided
+    input questions contexts pairs.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import ContextRelevanceEvaluator
+
+    questions = ["Who created the Python language?", "Why does Java needs a JVM?", "Is C++ better than Python?"]
+    contexts = [
+        [(
+            "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+            "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+            "programmers write clear, logical code for both small and large-scale software projects."
+        )],
+        [(
+            "Java is a high-level, class-based, object-oriented programming language that is designed to have as few "
+            "implementation dependencies as possible. The JVM has two primary functions: to allow Java programs to run"
+            "on any device or operating system (known as the 'write once, run anywhere' principle), and to manage and"
+            "optimize program memory."
+        )],
+        [(
+            "C++ is a general-purpose programming language created by Bjarne Stroustrup as an extension of the C "
+            "programming language."
+        )],
+    ]
+
+    evaluator = ContextRelevanceEvaluator()
+    result = evaluator.run(questions=questions, contexts=contexts)
+    print(result["score"])
+    # 0.67
+    print(result["individual_scores"])
+    # [1,1,0]
+    print(result["results"])
+    # [{
+    #   'relevant_statements': ['Python, created by Guido van Rossum in the late 1980s.'],
+    #    'score': 1.0
+    #  },
+    #  {
+    #   'relevant_statements': ['The JVM has two primary functions: to allow Java programs to run on any device or
+    #                           operating system (known as the "write once, run anywhere" principle), and to manage and
+    #                           optimize program memory'],
+    #   'score': 1.0
+    #  },
+    #  {
+    #   'relevant_statements': [],
+    #   'score': 0.0
+    #  }]
+    ```
+    """
+
+    def __init__(
+        self,
+        examples: list[dict[str, Any]] | None = None,
+        progress_bar: bool = True,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of ContextRelevanceEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param examples:
+            Optional few-shot examples conforming to the expected input and output format of ContextRelevanceEvaluator.
+            Default examples will be used if none are provided.
+            Each example must be a dictionary with keys "inputs" and "outputs".
+            "inputs" must be a dictionary with keys "questions" and "contexts".
+            "outputs" must be a dictionary with "relevant_statements".
+            Expected format:
+            ```python
+            [{
+                "inputs": {
+                    "questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."],
+                },
+                "outputs": {
+                    "relevant_statements": ["Rome is the capital of Italy."],
+                },
+            }]
+            ```
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param raise_on_failure:
+            Whether to raise an exception if the API call fails.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+
+        self.instructions = (
+            "Please extract only sentences from the provided context which are absolutely relevant and "
+            "required to answer the following question. If no relevant sentences are found, or if you "
+            "believe the question cannot be answered from the given context, return an empty list, example: []"
+        )
+        self.inputs = [("questions", list[str]), ("contexts", list[list[str]])]
+        self.outputs = ["relevant_statements"]
+        self.examples = examples or _DEFAULT_EXAMPLES
+
+        super(ContextRelevanceEvaluator, self).__init__(  # noqa: UP008
+            instructions=self.instructions,
+            inputs=self.inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=chat_generator,
+            raise_on_failure=raise_on_failure,
+            progress_bar=progress_bar,
+        )
+
+    @component.output_types(score=float, results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A list of lists of contexts. Each list of contexts corresponds to one question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean context relevance score over all the provided input questions.
+                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+        """
+        result = super(ContextRelevanceEvaluator, self).run(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    @component.output_types(score=float, results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A list of lists of contexts. Each list of contexts corresponds to one question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean context relevance score over all the provided input questions.
+                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+        """
+        result = await super(ContextRelevanceEvaluator, self).run_async(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
+        """
+        Post-processes raw LLM evaluator outputs to compute context relevance scores.
+
+        Calculates binary scores based on whether relevant statements were found,
+        averages the scores across all successful queries, and updates the result payload.
+
+        :param result:
+            The raw evaluation dictionary from the base LLM evaluator.
+        :returns:
+            The updated dictionary containing final scores and tracking metrics.
+        """
+        for idx, res in enumerate(result["results"]):
+            if res is None:
+                result["results"][idx] = {"relevant_statements": [], "score": float("nan")}
+                continue
+            if len(res["relevant_statements"]) > 0:
+                res["score"] = 1
+            else:
+                res["score"] = 0
+
+        # calculate average context relevance score over all queries
+        scores = [res["score"] for res in result["results"]]
+        valid_scores = [s for s in scores if not math.isnan(s)]
+        skipped = len(scores) - len(valid_scores)
+        if skipped:
+            logger.warning("{skipped} query(s) failed and were excluded from the score.", skipped=skipped)
+        result["score"] = mean(valid_scores) if valid_scores else float("nan")
+        result["individual_scores"] = scores  # useful for the EvaluationRunResult
+
+        return result
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            A dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            examples=self.examples,
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ContextRelevanceEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)

haystack/components/evaluators/document_map.py

@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentMAPEvaluator:
+    """
+    A Mean Average Precision (MAP) evaluator for documents.
+
+    Evaluator that calculates the mean average precision of the retrieved documents, a metric
+    that measures how high retrieved documents are ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMAPEvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentMAPEvaluator
+
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+
+    print(result["individual_scores"])
+    # [1.0, 0.8333333333333333]
+    print(result["score"])
+    # 0.9166666666666666
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentMAPEvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentMAPEvaluator on the given inputs.
+
+        All lists must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents
+                are ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True):
+            average_precision = 0.0
+            average_precision_numerator = 0.0
+            relevant_documents = 0
+
+            ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                retrieved_value = self._get_comparison_value(retrieved_document)
+                if retrieved_value is None:
+                    continue
+
+                if retrieved_value in ground_truth_values:
+                    relevant_documents += 1
+                    average_precision_numerator += relevant_documents / (rank + 1)
+            if relevant_documents > 0:
+                average_precision = average_precision_numerator / relevant_documents
+            individual_scores.append(average_precision)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+        return {"score": score, "individual_scores": individual_scores}

haystack/components/evaluators/document_mrr.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentMRREvaluator:
+    """
+    Evaluator that calculates the mean reciprocal rank of the retrieved documents.
+
+    MRR measures how high the first retrieved document is ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMRREvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentMRREvaluator
+
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+    print(result["individual_scores"])
+    # [1.0, 1.0]
+    print(result["score"])
+    # 1.0
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentMRREvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentMRREvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved
+                document is ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True):
+            reciprocal_rank = 0.0
+
+            ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                retrieved_value = self._get_comparison_value(retrieved_document)
+                if retrieved_value is None:
+                    continue
+                if retrieved_value in ground_truth_values:
+                    reciprocal_rank = 1 / (rank + 1)
+                    break
+            individual_scores.append(reciprocal_rank)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+
+        return {"score": score, "individual_scores": individual_scores}

haystack/components/evaluators/document_ndcg.py

@@ -0,0 +1,193 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from math import log2
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentNDCGEvaluator:
+    """
+    Evaluator that calculates the normalized discounted cumulative gain (NDCG) of retrieved documents.
+
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+    If the ground truth documents have relevance scores, the NDCG calculation uses these scores.
+    Otherwise, it assumes binary relevance of all ground truth documents.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentNDCGEvaluator
+
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="France", score=1.0), Document(content="Paris", score=0.5)]],
+        retrieved_documents=[[Document(content="France"), Document(content="Germany"), Document(content="Paris")]],
+    )
+    print(result["individual_scores"])
+    # [0.8869]
+    print(result["score"])
+    # 0.8869
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentNDCGEvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentNDCGEvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+        The list items within `ground_truth_documents` and `retrieved_documents` can differ in length.
+
+        :param ground_truth_documents:
+            Lists of expected documents, one list per question. Binary relevance is used if documents have no scores.
+        :param retrieved_documents:
+            Lists of retrieved documents, one list per question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the NDCG for each question.
+        """
+        self.validate_inputs(ground_truth_documents, retrieved_documents)
+
+        individual_scores = []
+
+        for gt_docs, ret_docs in zip(ground_truth_documents, retrieved_documents, strict=True):
+            dcg = self.calculate_dcg(gt_docs, ret_docs)
+            idcg = self.calculate_idcg(gt_docs)
+            ndcg = dcg / idcg if idcg > 0 else 0
+            individual_scores.append(ndcg)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+
+        return {"score": score, "individual_scores": individual_scores}
+
+    @staticmethod
+    def validate_inputs(gt_docs: list[list[Document]], ret_docs: list[list[Document]]) -> None:
+        """
+        Validate the input parameters.
+
+        :param gt_docs:
+            The ground_truth_documents to validate.
+        :param ret_docs:
+            The retrieved_documents to validate.
+
+        :raises ValueError:
+            If the ground_truth_documents or the retrieved_documents are an empty list.
+            If the length of ground_truth_documents and retrieved_documents differs.
+            If any list of documents in ground_truth_documents contains a mix of documents with and without a score.
+        """
+        if len(gt_docs) == 0 or len(ret_docs) == 0:
+            msg = "ground_truth_documents and retrieved_documents must be provided."
+            raise ValueError(msg)
+
+        if len(gt_docs) != len(ret_docs):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        for docs in gt_docs:
+            if any(doc.score is not None for doc in docs) and any(doc.score is None for doc in docs):
+                msg = "Either none or all documents in each list of ground_truth_documents must have a score."
+                raise ValueError(msg)
+
+    def calculate_dcg(self, gt_docs: list[Document], ret_docs: list[Document]) -> float:
+        """
+        Calculate the discounted cumulative gain (DCG) of the retrieved documents.
+
+        :param gt_docs:
+            The ground truth documents.
+        :param ret_docs:
+            The retrieved documents.
+        :returns:
+            The discounted cumulative gain (DCG) of the retrieved
+            documents based on the ground truth documents.
+        """
+        dcg = 0.0
+        # Build lookup from comparison value -> relevance score, skipping documents
+        # whose comparison value cannot be determined (e.g. missing meta key)
+        relevant_value_to_score: dict[Any, float] = {}
+        for doc in gt_docs:
+            value = self._get_comparison_value(doc)
+            if value is not None:
+                relevant_value_to_score[value] = doc.score if doc.score is not None else 1
+
+        for i, doc in enumerate(ret_docs):
+            value = self._get_comparison_value(doc)
+            if value is not None and value in relevant_value_to_score:
+                dcg += relevant_value_to_score[value] / log2(i + 2)  # i + 2 because i is 0-indexed
+        return dcg
+
+    def calculate_idcg(self, gt_docs: list[Document]) -> float:
+        """
+        Calculate the ideal discounted cumulative gain (IDCG) of the ground truth documents.
+
+        Ground truth documents whose comparison value cannot be determined (e.g. missing meta key)
+        are excluded, since they can never be matched in `calculate_dcg` either. Including them here
+        would inflate the IDCG and make it impossible for NDCG to reach 1.0 for a perfect retrieval.
+
+        :param gt_docs:
+            The ground truth documents.
+        :returns:
+            The ideal discounted cumulative gain (IDCG) of the ground truth documents.
+        """
+        # Filter out documents that cannot be matched, consistent with calculate_dcg
+        matchable_docs = [doc for doc in gt_docs if self._get_comparison_value(doc) is not None]
+
+        idcg = 0.0
+        for i, doc in enumerate(
+            sorted(matchable_docs, key=lambda x: x.score if x.score is not None else 1, reverse=True)
+        ):
+            # If the document has a score, use it; otherwise, use 1 for binary relevance.
+            relevance = doc.score if doc.score is not None else 1
+            idcg += relevance / log2(i + 2)  # i + 2 because i is 0-indexed
+        return idcg

haystack/components/evaluators/document_recall.py

@@ -0,0 +1,181 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from typing import Any
+
+from haystack import component, default_to_dict, logging
+from haystack.dataclasses import Document
+
+logger = logging.getLogger(__name__)
+
+
+class RecallMode(Enum):
+    """
+    Enum for the mode to use for calculating the recall score.
+    """
+
+    # Score is based on whether any document is retrieved.
+    SINGLE_HIT = "single_hit"
+    # Score is based on how many documents were retrieved.
+    MULTI_HIT = "multi_hit"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "RecallMode":
+        """
+        Convert a string to a RecallMode enum.
+        """
+        enum_map = {e.value: e for e in RecallMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown recall mode '{string}'. Supported modes are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class DocumentRecallEvaluator:
+    """
+    Evaluator that calculates the Recall score for a list of documents.
+
+    Returns both a list of scores for each question and the average.
+    There can be multiple ground truth documents and multiple predicted documents as input.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentRecallEvaluator
+
+    evaluator = DocumentRecallEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+    print(result["individual_scores"])
+    # [1.0, 1.0]
+    print(result["score"])
+    # 1.0
+    ```
+    """
+
+    def __init__(
+        self, mode: str | RecallMode = RecallMode.SINGLE_HIT, document_comparison_field: str = "content"
+    ) -> None:
+        """
+        Create a DocumentRecallEvaluator component.
+
+        :param mode:
+            Mode to use for calculating the recall score.
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        if isinstance(mode, str):
+            mode = RecallMode.from_str(mode)
+
+        self.mode = mode
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
+        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        return float(len(retrieved_ground_truths) > 0)
+
+    def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
+        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        if not unique_truths or unique_truths <= {"", None}:
+            logger.warning(
+                "There are no ground truth documents or none of them contain a valid comparison value. "
+                "Score will be set to 0."
+            )
+            return 0.0
+
+        if not unique_retrievals or unique_retrievals <= {"", None}:
+            logger.warning(
+                "There are no retrieved documents or none of them contain a valid comparison value. "
+                "Score will be set to 0."
+            )
+            return 0.0
+
+        return len(retrieved_ground_truths) / len(unique_truths)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentRecallEvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching
+                documents retrieved. If the mode is `single_hit`, the individual scores are 0 or 1.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        if self.mode == RecallMode.SINGLE_HIT:
+            mode_function = self._recall_single_hit
+        elif self.mode == RecallMode.MULTI_HIT:
+            mode_function = self._recall_multi_hit
+
+        scores = [mode_function(gt, ret) for gt, ret in zip(ground_truth_documents, retrieved_documents, strict=True)]
+
+        return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, mode=str(self.mode), document_comparison_field=self.document_comparison_field)

haystack/components/evaluators/faithfulness.py

@@ -0,0 +1,255 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+from typing import Any
+
+from numpy import mean as np_mean
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.evaluators.llm_evaluator import LLMEvaluator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.utils import deserialize_chatgenerator_inplace
+
+logger = logging.getLogger(__name__)
+
+# Default examples to include in the prompt if the user does not provide any examples
+_DEFAULT_EXAMPLES = [
+    {
+        "inputs": {
+            "questions": "What is the capital of Germany and when was it founded?",
+            "contexts": ["Berlin is the capital of Germany and was founded in 1244."],
+            "predicted_answers": "The capital of Germany, Berlin, was founded in the 13th century.",
+        },
+        "outputs": {
+            "statements": ["Berlin is the capital of Germany.", "Berlin was founded in 1244."],
+            "statement_scores": [1, 1],
+        },
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of France?",
+            "contexts": ["Berlin is the capital of Germany."],
+            "predicted_answers": "Paris",
+        },
+        "outputs": {"statements": ["Paris is the capital of France."], "statement_scores": [0]},
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of Italy?",
+            "contexts": ["Rome is the capital of Italy."],
+            "predicted_answers": "Rome is the capital of Italy with more than 4 million inhabitants.",
+        },
+        "outputs": {
+            "statements": ["Rome is the capital of Italy.", "Rome has more than 4 million inhabitants."],
+            "statement_scores": [1, 0],
+        },
+    },
+]
+
+
+@component
+class FaithfulnessEvaluator(LLMEvaluator):
+    """
+    Evaluator that checks if a generated answer can be inferred from the provided contexts.
+
+    An LLM separates the answer into multiple statements and checks whether the statement can be inferred from the
+    context or not. The final score for the full answer is a number from 0.0 to 1.0. It represents the proportion of
+    statements that can be inferred from the provided contexts.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import FaithfulnessEvaluator
+
+    questions = ["Who created the Python language?"]
+    contexts = [
+        [(
+            "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+            "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+            "programmers write clear, logical code for both small and large-scale software projects."
+        )],
+    ]
+    predicted_answers = [
+        "Python is a high-level general-purpose programming language that was created by George Lucas."
+    ]
+    evaluator = FaithfulnessEvaluator()
+    result = evaluator.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+
+    print(result["individual_scores"])
+    # [0.5]
+    print(result["score"])
+    # 0.5
+    print(result["results"])
+    # [{'statements': ['Python is a high-level general-purpose programming language.',
+    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5}]
+    ```
+    """
+
+    def __init__(
+        self,
+        examples: list[dict[str, Any]] | None = None,
+        progress_bar: bool = True,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of FaithfulnessEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param examples:
+            Optional few-shot examples conforming to the expected input and output format of FaithfulnessEvaluator.
+            Default examples will be used if none are provided.
+            Each example must be a dictionary with keys "inputs" and "outputs".
+            "inputs" must be a dictionary with keys "questions", "contexts", and "predicted_answers".
+            "outputs" must be a dictionary with "statements" and "statement_scores".
+            Expected format:
+            ```python
+            [{
+                "inputs": {
+                    "questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."],
+                    "predicted_answers": "Rome is the capital of Italy with more than 4 million inhabitants.",
+                },
+                "outputs": {
+                    "statements": ["Rome is the capital of Italy.", "Rome has more than 4 million inhabitants."],
+                    "statement_scores": [1, 0],
+                },
+            }]
+            ```
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param raise_on_failure:
+            Whether to raise an exception if the API call fails.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+        self.instructions = (
+            "Your task is to judge the faithfulness or groundedness of statements based "
+            "on context information. First, please extract statements from a provided "
+            "predicted answer to a question. Second, calculate a faithfulness score for each "
+            "statement made in the predicted answer. The score is 1 if the statement can be "
+            "inferred from the provided context or 0 if it cannot be inferred."
+        )
+        self.inputs = [("questions", list[str]), ("contexts", list[list[str]]), ("predicted_answers", list[str])]
+        self.outputs = ["statements", "statement_scores"]
+        self.examples = examples or _DEFAULT_EXAMPLES
+
+        super(FaithfulnessEvaluator, self).__init__(  # noqa: UP008
+            instructions=self.instructions,
+            inputs=self.inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=chat_generator,
+            raise_on_failure=raise_on_failure,
+            progress_bar=progress_bar,
+        )
+
+    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A nested list of contexts that correspond to the questions.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean faithfulness score over all the provided input answers.
+                - `individual_scores`: A list of faithfulness scores for each input answer.
+                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+        """
+        result = super(FaithfulnessEvaluator, self).run(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A nested list of contexts that correspond to the questions.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean faithfulness score over all the provided input answers.
+                - `individual_scores`: A list of faithfulness scores for each input answer.
+                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+        """
+        result = await super(FaithfulnessEvaluator, self).run_async(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
+        """
+        Post-processes raw LLM evaluator outputs to compute faithfulness scores.
+
+        Calculates statement-level score averages, computes the overall mean faithfulness
+        score across successful queries, and updates the result payload.
+
+        :param result:
+            The raw evaluation dictionary from the base LLM evaluator.
+        :returns:
+            The updated dictionary containing final scores and tracking metrics.
+        """
+
+        # calculate average statement faithfulness score per query
+        for idx, res in enumerate(result["results"]):
+            if res is None:
+                result["results"][idx] = {"statements": [], "statement_scores": [], "score": float("nan")}
+                continue
+            if not res["statements"]:
+                res["score"] = 0
+            else:
+                res["score"] = np_mean(res["statement_scores"])
+
+        # calculate average answer faithfulness score over all queries
+        scores = [res["score"] for res in result["results"]]
+        valid_scores = [s for s in scores if not math.isnan(s)]
+        skipped = len(scores) - len(valid_scores)
+        if skipped:
+            logger.warning("{skipped} query(s) failed and were excluded from the score.", skipped=skipped)
+        result["score"] = np_mean(valid_scores) if valid_scores else float("nan")
+        result["individual_scores"] = scores
+
+        return result
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            A dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            examples=self.examples,
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "FaithfulnessEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)

haystack/components/evaluators/llm_evaluator.py

@@ -0,0 +1,462 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+from typing import Any
+
+from tqdm import tqdm
+from tqdm.asyncio import tqdm as async_tqdm
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace, deserialize_type, serialize_type
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLMEvaluator:
+    """
+    Uses an LLM to evaluate inputs based on a prompt containing instructions and examples.
+
+    The default API requires an OpenAI API key to be provided as an environment variable "OPENAI_API_KEY".
+    The inputs are lists that are user-defined depending on the desired metric.
+    The output is a dictionary with a key `results` containing a list of evaluation results.
+    Each result is a dictionary with user-defined keys and values of either 0 for FALSE or 1 for TRUE respectively.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import LLMEvaluator
+    evaluator = LLMEvaluator(
+        instructions="Is this answer problematic for children?",
+        inputs=[("predicted_answers", list[str])],
+        outputs=["score"],
+        examples=[
+            {"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}},
+        ],
+    )
+    predicted_answers = [
+        "Football is the most popular sport with around 4 billion followers worldwide",
+        "Python language was created by Guido van Rossum.",
+    ]
+    results = evaluator.run(predicted_answers=predicted_answers)
+    print(results)
+    # {'results': [{'score': 0}, {'score': 0}]}
+    ```
+    """
+
+    def __init__(
+        self,
+        instructions: str,
+        inputs: list[tuple[str, type[list]]],
+        outputs: list[str],
+        examples: list[dict[str, Any]],
+        progress_bar: bool = True,
+        *,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of LLMEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param instructions:
+            The prompt instructions to use for evaluation.
+            Should be a question about the inputs that can be answered with yes or no.
+        :param inputs:
+            The inputs that the component expects as incoming connections and that it evaluates.
+            Each input is a tuple of an input name and input type. Input types must be lists.
+        :param outputs:
+            Output names of the evaluation results. They correspond to keys in the output dictionary.
+        :param examples:
+            Few-shot examples conforming to the expected input and output format as defined in the `inputs` and
+             `outputs` parameters.
+            Each example is a dictionary with keys "inputs" and "outputs"
+            They contain the input and output as dictionaries respectively.
+        :param raise_on_failure:
+            If True, the component will raise an exception on an unsuccessful API call.
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+        self.validate_init_parameters(inputs, outputs, examples)
+        component.set_input_types(self, **dict(inputs))
+
+        self.raise_on_failure = raise_on_failure
+        self.instructions = instructions
+        self.inputs = inputs
+        self.outputs = outputs
+        self.examples = examples
+        self.progress_bar = progress_bar
+
+        template = self.prepare_template()
+        self.builder = PromptBuilder(template=template)
+
+        if chat_generator is not None:
+            self._chat_generator = chat_generator
+        else:
+            generation_kwargs = {"response_format": {"type": "json_object"}, "seed": 42}
+            self._chat_generator = OpenAIChatGenerator(generation_kwargs=generation_kwargs)
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator.
+        """
+        if hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator on the serving event loop.
+        """
+        if hasattr(self._chat_generator, "warm_up_async"):
+            await self._chat_generator.warm_up_async()
+        elif hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's resources.
+        """
+        if hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's async resources.
+        """
+        if hasattr(self._chat_generator, "close_async"):
+            await self._chat_generator.close_async()
+        elif hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    @staticmethod
+    def validate_init_parameters(
+        inputs: list[tuple[str, type[list]]], outputs: list[str], examples: list[dict[str, Any]]
+    ) -> None:
+        """
+        Validate the init parameters.
+
+        :param inputs:
+            The inputs to validate.
+        :param outputs:
+            The outputs to validate.
+        :param examples:
+            The examples to validate.
+
+        :raises ValueError:
+            If the inputs are not a list of tuples with a string and a type of list.
+            If the outputs are not a list of strings.
+            If the examples are not a list of dictionaries.
+            If any example does not have keys "inputs" and "outputs" with values that are dictionaries with string keys.
+        """
+        # Validate inputs
+        if (
+            not isinstance(inputs, list)
+            or not all(isinstance(_input, tuple) for _input in inputs)
+            or not all(isinstance(_input[0], str) and _input[1] is not list and len(_input) == 2 for _input in inputs)
+        ):
+            msg = (
+                f"LLM evaluator expects inputs to be a list of tuples. Each tuple must contain an input name and "
+                f"type of list but received {inputs}."
+            )
+            raise ValueError(msg)
+
+        # Validate outputs
+        if not isinstance(outputs, list) or not all(isinstance(output, str) for output in outputs):
+            msg = f"LLM evaluator expects outputs to be a list of str but received {outputs}."
+            raise ValueError(msg)
+
+        # Validate examples are lists of dicts
+        if not isinstance(examples, list) or not all(isinstance(example, dict) for example in examples):
+            msg = f"LLM evaluator expects examples to be a list of dictionaries but received {examples}."
+            raise ValueError(msg)
+
+        # Validate each example
+        for example in examples:
+            if (
+                {"inputs", "outputs"} != example.keys()
+                or not all(isinstance(example[param], dict) for param in ["inputs", "outputs"])
+                or not all(isinstance(key, str) for param in ["inputs", "outputs"] for key in example[param])
+            ):
+                msg = (
+                    f"LLM evaluator expects each example to have keys `inputs` and `outputs` with values that are "
+                    f"dictionaries with str keys but received {example}."
+                )
+                raise ValueError(msg)
+
+    @component.output_types(results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param inputs:
+            The input values to evaluate. The keys are the input names and the values are lists of input values.
+        :returns:
+            A dictionary with a `results` entry that contains a list of results.
+            Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
+            and the evaluation results as the values. If an exception occurs for a particular input value, the result
+            will be `None` for that entry.
+            If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
+            in the output dictionary, under the key "meta".
+        :raises ValueError:
+            Only in the case that  `raise_on_failure` is set to True and the received inputs are not lists or have
+            different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
+        """
+        self.warm_up()
+
+        self.validate_input_parameters(dict(self.inputs), inputs)
+
+        # inputs is a dictionary with keys being input names and values being a list of input values
+        # We need to iterate through the lists in parallel for all keys of the dictionary
+        input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
+        list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
+
+        results: list[dict[str, Any] | None] = []
+        metadata = []
+        errors = 0
+        for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
+            prompt = self.builder.run(**input_names_to_values)
+            messages = [ChatMessage.from_user(prompt["prompt"])]
+            try:
+                result = self._chat_generator.run(messages=messages)
+            except Exception as e:
+                if self.raise_on_failure:
+                    raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
+                logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
+                results.append(None)
+                errors += 1
+                continue
+
+            parsed_result = _parse_dict_from_json(
+                result["replies"][0].text, expected_keys=self.outputs, raise_on_failure=self.raise_on_failure
+            )
+            if parsed_result is None:
+                results.append(None)
+                errors += 1
+            else:
+                results.append(parsed_result)
+
+            if result["replies"][0].meta:
+                metadata.append(result["replies"][0].meta)
+
+        if errors > 0:
+            logger.warning(
+                "LLM evaluator failed for {errors} out of {len(list_of_input_names_to_values)} inputs.",
+                errors=errors,
+                len=len(list_of_input_names_to_values),
+            )
+
+        return {"results": results, "meta": metadata or None}
+
+    @component.output_types(results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously
+
+        :param inputs:
+            The input values to evaluate. The keys are the input names and the values are lists of input values.
+        :returns:
+            A dictionary with a `results` entry that contains a list of results.
+            Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
+            and the evaluation results as the values. If an exception occurs for a particular input value, the result
+            will be `None` for that entry.
+            If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
+            in the output dictionary, under the key "meta".
+        :raises TypeError:
+            If the chat generator does not support async execution.
+        :raises ValueError:
+            Only in the case that  `raise_on_failure` is set to True and the received inputs are not lists or have
+            different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
+        """
+
+        await self.warm_up_async()
+
+        self.validate_input_parameters(dict(self.inputs), inputs)
+
+        # inputs is a dictionary with keys being input names and values being a list of input values
+        # We need to iterate through the lists in parallel for all keys of the dictionary
+        input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
+        list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
+
+        results: list[dict[str, Any] | None] = []
+        metadata = []
+        errors = 0
+
+        generator_has_async = hasattr(self._chat_generator, "run_async")
+        for input_names_to_values in async_tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
+            prompt = self.builder.run(**input_names_to_values)
+            messages = [ChatMessage.from_user(prompt["prompt"])]
+            try:
+                if generator_has_async:
+                    result = await self._chat_generator.run_async(messages=messages)  # type: ignore[attr-defined]
+                else:
+                    logger.debug(
+                        "{generator_type} does not implement 'run_async'."
+                        " Running the synchronous 'run' method in a thread to avoid blocking the event loop.",
+                        generator_type=type(self._chat_generator).__name__,
+                    )
+                    result = await asyncio.to_thread(self._chat_generator.run, messages=messages)
+            except Exception as e:
+                if self.raise_on_failure:
+                    raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
+                logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
+                results.append(None)
+                errors += 1
+                continue
+
+            parsed_result = _parse_dict_from_json(
+                result["replies"][0].text, expected_keys=self.outputs, raise_on_failure=self.raise_on_failure
+            )
+            if parsed_result is None:
+                results.append(None)
+                errors += 1
+            else:
+                results.append(parsed_result)
+
+            if result["replies"][0].meta:
+                metadata.append(result["replies"][0].meta)
+
+        if errors > 0:
+            logger.warning(
+                "LLM evaluator failed for {errors} out of {len(list_of_input_names_to_values)} inputs.",
+                errors=errors,
+                len=len(list_of_input_names_to_values),
+            )
+
+        return {"results": results, "meta": metadata or None}
+
+    def prepare_template(self) -> str:
+        """
+        Prepare the prompt template.
+
+        Combine instructions, inputs, outputs, and examples into one prompt template with the following format:
+        Instructions:
+        `<instructions>`
+
+        Generate the response in JSON format with the following keys:
+        `<list of output keys>`
+        Consider the instructions and the examples below to determine those values.
+
+        Examples:
+        `<examples>`
+
+        Inputs:
+        `<inputs>`
+        Outputs:
+
+        :returns:
+            The prompt template.
+        """
+        inputs_section = (
+            "{" + ", ".join([f'"{input_socket[0]}": {{{{ {input_socket[0]} }}}}' for input_socket in self.inputs]) + "}"
+        )
+
+        examples_section = "\n".join(
+            [
+                "Inputs:\n" + json.dumps(example["inputs"]) + "\nOutputs:\n" + json.dumps(example["outputs"])
+                for example in self.examples
+            ]
+        )
+        return (
+            f"Instructions:\n"
+            f"{self.instructions}\n\n"
+            f"Generate the response in JSON format with the following keys:\n"
+            f"{json.dumps(self.outputs)}\n"
+            f"Consider the instructions and the examples below to determine those values.\n\n"
+            f"Examples:\n"
+            f"{examples_section}\n\n"
+            f"Inputs:\n"
+            f"{inputs_section}\n"
+            f"Outputs:\n"
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        # Since we cannot currently serialize tuples, convert the inputs to a list.
+        inputs = [[name, serialize_type(type_)] for name, type_ in self.inputs]
+        return default_to_dict(
+            self,
+            instructions=self.instructions,
+            inputs=inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        data["init_parameters"]["inputs"] = [
+            (name, deserialize_type(type_)) for name, type_ in data["init_parameters"]["inputs"]
+        ]
+
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+
+        return default_from_dict(cls, data)
+
+    @staticmethod
+    def validate_input_parameters(expected: dict[str, Any], received: dict[str, Any]) -> None:
+        """
+        Validate the input parameters.
+
+        :param expected:
+            The expected input parameters.
+        :param received:
+            The received input parameters.
+
+        :raises ValueError:
+            If not all expected inputs are present in the received inputs
+            If the received inputs are not lists or have different lengths
+        """
+        # Validate that all expected inputs are present in the received inputs
+        for param in expected:
+            if param not in received:
+                msg = f"LLM evaluator expected input parameter '{param}' but received only {received.keys()}."
+                raise ValueError(msg)
+
+        # Validate that all received inputs are lists
+        if not all(isinstance(_input, list) for _input in received.values()):
+            msg = (
+                "LLM evaluator expects all input values to be lists but received "
+                f"{[type(_input) for _input in received.values()]}."
+            )
+            raise ValueError(msg)
+
+        # Validate that all received inputs are of the same length
+        inputs = received.values()
+        length = len(next(iter(inputs)))
+        if not all(len(_input) == length for _input in inputs):
+            msg = (
+                f"LLM evaluator expects all input lists to have the same length but received {inputs} with lengths "
+                f"{[len(_input) for _input in inputs]}."
+            )
+            raise ValueError(msg)

haystack/components/evaluators/sas_evaluator.py

@@ -0,0 +1,188 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from numpy import mean as np_mean
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.lazy_imports import LazyImport
+from haystack.utils import ComponentDevice, expit
+from haystack.utils.auth import Secret
+
+with LazyImport(message="Run 'pip install \"sentence-transformers>=5.0.0\"'") as sas_import:
+    from sentence_transformers import CrossEncoder, SentenceTransformer, util
+    from transformers import AutoConfig
+
+
+@component
+class SASEvaluator:
+    """
+    SASEvaluator computes the Semantic Answer Similarity (SAS) between a list of predictions and a one of ground truths.
+
+    It's usually used in Retrieval Augmented Generation (RAG) pipelines to evaluate the quality of the generated
+    answers. The SAS is computed using a pre-trained model from the Hugging Face model hub. The model can be either a
+    Bi-Encoder or a Cross-Encoder. The choice of the model is based on the `model` parameter.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators.sas_evaluator import SASEvaluator
+
+    evaluator = SASEvaluator(model="cross-encoder/ms-marco-MiniLM-L-6-v2")
+    ground_truths = [
+        "A construction budget of US $2.3 billion",
+        "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+        "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+    ]
+    predictions = [
+        "A construction budget of US $2.3 billion",
+        "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+        "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+    ]
+    result = evaluator.run(
+        ground_truth_answers=ground_truths, predicted_answers=predictions
+    )
+
+    print(result["score"])
+    # 0.9999673763910929
+
+    print(result["individual_scores"])
+    # [0.9999765157699585, 0.999968409538269, 0.9999572038650513]
+    ```
+    """
+
+    def __init__(
+        self,
+        model: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+        batch_size: int = 32,
+        device: ComponentDevice | None = None,
+        token: Secret = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
+    ) -> None:
+        """
+        Creates a new instance of SASEvaluator.
+
+        :param model:
+            SentenceTransformers semantic textual similarity model, should be path or string pointing to a downloadable
+            model.
+        :param batch_size:
+            Number of prediction-label pairs to encode at once.
+        :param device:
+            The device on which the model is loaded. If `None`, the default device is automatically selected.
+        :param token:
+            The Hugging Face token for HTTP bearer authorization.
+            You can find your HF token in your [account settings](https://huggingface.co/settings/tokens)
+        """
+        sas_import.check()
+
+        self._model = model
+        self._batch_size = batch_size
+        self._device = device
+        self._token = token
+        self._similarity_model: SentenceTransformer | CrossEncoder | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        return default_to_dict(
+            self, model=self._model, batch_size=self._batch_size, device=self._device, token=self._token
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SASEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """
+        Initializes the component.
+        """
+        if self._similarity_model:
+            return
+
+        token = self._token.resolve_value() if self._token else None
+        config = AutoConfig.from_pretrained(self._model, use_auth_token=token)
+        cross_encoder_used = False
+        if config.architectures:
+            cross_encoder_used = any(arch.endswith("ForSequenceClassification") for arch in config.architectures)
+        device = ComponentDevice.resolve_device(self._device).to_torch_str()
+        # Based on the Model string we can load either Bi-Encoders or Cross Encoders.
+        # Similarity computation changes for both approaches
+        if cross_encoder_used:
+            self._similarity_model = CrossEncoder(self._model, device=device, token=token)
+        else:
+            self._similarity_model = SentenceTransformer(self._model, device=device, token=token)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, float | list[float]]:
+        """
+        SASEvaluator component run method.
+
+        Run the SASEvaluator to compute the Semantic Answer Similarity (SAS) between a list of predicted answers
+        and a list of ground truth answers. Both must be list of strings of same length.
+
+        :param ground_truth_answers:
+            A list of expected answers for each question.
+        :param predicted_answers:
+            A list of generated answers for each question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean SAS score over all the predictions/ground-truth pairs.
+                - `individual_scores`: A list of similarity scores for each prediction/ground-truth pair.
+        """
+        if len(ground_truth_answers) != len(predicted_answers):
+            raise ValueError("The number of predictions and labels must be the same.")
+
+        if any(answer is None for answer in predicted_answers):
+            raise ValueError("Predicted answers must not contain None values.")
+
+        if len(predicted_answers) == 0:
+            return {"score": 0.0, "individual_scores": [0.0]}
+
+        if not self._similarity_model:
+            self.warm_up()
+
+        if isinstance(self._similarity_model, CrossEncoder):
+            # For Cross Encoders we create a list of pairs of predictions and labels
+            sentence_pairs = list(zip(predicted_answers, ground_truth_answers, strict=True))
+            similarity_scores = self._similarity_model.predict(
+                sentence_pairs, batch_size=self._batch_size, convert_to_numpy=True
+            )
+
+            # All Cross Encoders do not return a set of logits scores that are normalized
+            # We normalize scores if they are larger than 1
+            if (similarity_scores > 1).any():
+                similarity_scores = expit(similarity_scores)
+
+            # Convert scores to list of floats from numpy array
+            similarity_scores = similarity_scores.tolist()
+
+        elif isinstance(self._similarity_model, SentenceTransformer):
+            # For Bi-encoders we create embeddings separately for predictions and labels
+            predictions_embeddings = self._similarity_model.encode(
+                predicted_answers, batch_size=self._batch_size, convert_to_tensor=True
+            )
+            label_embeddings = self._similarity_model.encode(
+                ground_truth_answers, batch_size=self._batch_size, convert_to_tensor=True
+            )
+
+            # Compute cosine-similarities
+            similarity_scores = [
+                float(util.cos_sim(pred_embedding, label_embedding).cpu().squeeze().numpy())
+                for pred_embedding, label_embedding in zip(predictions_embeddings, label_embeddings, strict=True)
+            ]
+
+        sas_score = np_mean(similarity_scores)
+
+        return {"score": sas_score, "individual_scores": similarity_scores}

haystack/components/extractors/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"llm_metadata_extractor": ["LLMMetadataExtractor"], "regex_text_extractor": ["RegexTextExtractor"]}
+
+if TYPE_CHECKING:
+    from .llm_metadata_extractor import LLMMetadataExtractor as LLMMetadataExtractor
+    from .regex_text_extractor import RegexTextExtractor as RegexTextExtractor
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/extractors/image/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"llm_document_content_extractor": ["LLMDocumentContentExtractor"]}
+
+if TYPE_CHECKING:
+    from .llm_document_content_extractor import LLMDocumentContentExtractor as LLMDocumentContentExtractor
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/extractors/image/llm_document_content_extractor.py

@@ -0,0 +1,422 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from typing import Any, Literal
+
+from jinja2 import meta
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.image.document_to_image import DocumentToImageContent
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ImageContent, TextContent
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+# Reserved key in the LLM JSON response that holds the main document text.
+DOCUMENT_CONTENT_KEY = "document_content"
+
+
+DEFAULT_PROMPT_TEMPLATE = """
+You are part of an information extraction pipeline that extracts the content of image-based documents.
+
+Extract the content from the provided image.
+You need to extract the content exactly.
+Format everything as markdown.
+Make sure to retain the reading order of the document.
+
+**Visual Elements**
+Do not extract figures, drawings, maps, graphs or any other visual elements.
+Instead, add a caption that describes briefly what you see in the visual element.
+You must describe each visual element.
+If you only see a visual element without other content, you must describe this visual element.
+Enclose each image caption with [img-caption][/img-caption]
+
+**Tables**
+Make sure to format the table in markdown.
+Add a short caption below the table that describes the table's content.
+Enclose each table caption with [table-caption][/table-caption].
+The caption must be placed below the extracted table.
+
+**Forms**
+Reproduce checkbox selections with markdown.
+
+Return a single JSON object. It must contain the key "document_content" with the extracted text as value.
+
+No markdown, no code fence, only raw JSON.
+
+Document:"""
+
+
+@component
+class LLMDocumentContentExtractor:
+    """
+    Extracts textual content and optionally metadata from image-based documents using a vision-enabled LLM.
+
+    One prompt and one LLM call per document. The component converts each document to an image via
+    DocumentToImageContent and sends it to the ChatGenerator. The prompt must not contain Jinja variables.
+
+    Response handling:
+    - If the LLM returns a **plain string** (non-JSON or not a JSON object), it is written to the document's content.
+    - If the LLM returns a **JSON object with only the key** `document_content`, that value is written to content.
+    - If the LLM returns a **JSON object with multiple keys**, the value of ``document_content`` (if present) is
+      written to content and all other keys are merged into the document's metadata.
+
+    The ChatGenerator can be configured to return JSON (e.g. ``response_format={"type": "json_object"}``
+    in ``generation_kwargs``).
+
+    Documents that fail extraction are returned in ``failed_documents`` with ``content_extraction_error`` in metadata.
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.extractors.image import LLMDocumentContentExtractor
+
+    prompt = \"\"\"
+    Extract the content from the provided image.
+    Format everything as markdown. Return only the extracted content as a JSON object with the key 'document_content'.
+    No markdown, no code fence, only raw JSON.
+
+    Extract metadata about the image like source of the image, date of creation, etc. if you can.
+    Return this metadata as additional key-value pairs in the same JSON object.
+    \"\"\"
+
+    chat_generator = OpenAIChatGenerator(
+            generation_kwargs={
+                    "response_format": {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": "entity_extraction",
+                            "schema": {
+                                "type": "object",
+                                "properties": {
+                                    "document_content": {"type": "string"},
+                                    "author": {"type": "string"},
+                                    "date": {"type": "string"},
+                                    "document_type": {"type": "string"},
+                                    "title": {"type": "string"},
+                                },
+                                "additionalProperties": False,
+                            },
+                        },
+                    }
+                }
+            )
+
+    extractor = LLMDocumentContentExtractor(
+        chat_generator=chat_generator,
+        file_path_meta_field="file_path",
+        raise_on_failure=False
+    )
+
+    documents = [
+        Document(content="", meta={"file_path": "test/test_files/images/image_metadata.png"}),
+        Document(content="", meta={"file_path": "test/test_files/images/apple.jpg", "page_number": 1})
+    ]
+    result = extractor.run(documents=documents)
+    updated_documents = result["documents"]
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator,
+        prompt: str = DEFAULT_PROMPT_TEMPLATE,
+        file_path_meta_field: str = "file_path",
+        root_path: str | None = None,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+        raise_on_failure: bool = False,
+        max_workers: int = 3,
+    ) -> None:
+        """
+        Initialize the LLMDocumentContentExtractor component.
+
+        :param chat_generator: A ChatGenerator that supports vision input. Optionally configured for JSON
+            (e.g. ``response_format={"type": "json_object"}`` in ``generation_kwargs``).
+        :param prompt: Prompt for extraction. Must not contain Jinja variables.
+        :param file_path_meta_field: The metadata field in the Document that contains the file path to the image or PDF.
+        :param root_path: The root directory path where document files are located. If provided, file paths in
+            document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
+        :param detail: Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
+        :param size: If provided, resizes the image to fit within (width, height) while keeping aspect ratio.
+        :param raise_on_failure: If True, exceptions from the LLM are raised. If False, failed documents are returned.
+        :param max_workers: Maximum number of threads for parallel LLM calls.
+        """
+        self._chat_generator = chat_generator
+        self.prompt = prompt
+        self.file_path_meta_field = file_path_meta_field
+        self.root_path = root_path or ""
+        self.detail = detail
+        self.size = size
+        LLMDocumentContentExtractor._validate_prompt_no_variables(prompt)
+        self.raise_on_failure = raise_on_failure
+        self.max_workers = max_workers
+        self._document_to_image_content = DocumentToImageContent(
+            file_path_meta_field=file_path_meta_field, root_path=root_path, detail=detail, size=size
+        )
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator.
+        """
+        if hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator on the serving event loop.
+        """
+        if hasattr(self._chat_generator, "warm_up_async"):
+            await self._chat_generator.warm_up_async()
+        elif hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's resources.
+        """
+        if hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's async resources.
+        """
+        if hasattr(self._chat_generator, "close_async"):
+            await self._chat_generator.close_async()
+        elif hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            prompt=self.prompt,
+            file_path_meta_field=self.file_path_meta_field,
+            root_path=self.root_path,
+            detail=self.detail,
+            size=self.size,
+            raise_on_failure=self.raise_on_failure,
+            max_workers=self.max_workers,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMDocumentContentExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary with serialized data.
+        :returns:
+            An instance of the component.
+        """
+        init_params = data.get("init_parameters", {})
+        deserialize_chatgenerator_inplace(init_params, key="chat_generator")
+
+        return default_from_dict(cls, data)
+
+    @staticmethod
+    def _validate_prompt_no_variables(prompt: str) -> None:
+        ast = SandboxedEnvironment().parse(prompt)
+        template_variables = meta.find_undeclared_variables(ast)
+        variables = list(template_variables)
+        if variables:
+            raise ValueError(
+                f"The prompt must not have any variables, only instructions on how to extract the content of the "
+                f"the image-based document. Found {','.join(variables)} in the prompt."
+            )
+
+    @staticmethod
+    def _process_response(response_text: str) -> tuple[str | None, dict[str, Any], str | None]:
+        """
+        Parse LLM response. Returns (content, meta_updates, error).
+
+        - Plain string (non-JSON): use entire response as document content;
+        - Valid JSON object: use key ``document_content`` for Document.content and all other keys for Document.metadata;
+        - Valid JSON but not an object (e.g. array or primitive), report an error;
+        """
+        try:
+            parsed = _parse_dict_from_json(response_text, raise_on_failure=True)
+        except json.JSONDecodeError:
+            return response_text, {}, None
+        except ValueError:
+            return None, {}, "Response must be a JSON object, not an array or primitive."
+
+        content = parsed.get(DOCUMENT_CONTENT_KEY)
+        meta_updates = {k: v for k, v in parsed.items() if k != DOCUMENT_CONTENT_KEY}
+        return content, meta_updates, None
+
+    def _run_on_thread(self, image_content: ImageContent | None) -> dict[str, Any]:
+        """
+        Execute the LLM inference in a separate thread for each document.
+
+        :param image_content: The image content for one document, or None if conversion failed.
+        :returns:
+            The LLM response if successful, or a dictionary with an "error" key on failure.
+        """
+        if image_content is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        # the prompt is the same for all documents, so we can set it up once here for each document/thread
+        message = ChatMessage.from_user(content_parts=[TextContent(text=self.prompt), image_content])
+
+        try:
+            result = self._chat_generator.run(messages=[message])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+
+        return result
+
+    async def _run_async(self, image_content: ImageContent | None) -> dict[str, Any]:
+        """
+        Execute the LLM inference asynchronously for each document.
+
+        :param image_content: The image content for one document, or None if conversion failed.
+        :returns:
+            The LLM response if successful, or a dictionary with an "error" key on failure.
+        """
+        if image_content is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        # the prompt is the same for all documents, so we can set it up once here for each document
+        message = ChatMessage.from_user(content_parts=[TextContent(text=self.prompt), image_content])
+
+        try:
+            result = await _execute_component_async(self._chat_generator, messages=[message])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+
+        return result
+
+    @staticmethod
+    def _process_llm_results(document: Document, result: dict[str, Any]) -> tuple[Document, bool]:
+        """
+        Process one document's LLM result using the unified response logic.
+
+        Returns (updated_document, True if success else False).
+        """
+        if "error" in result:
+            new_meta = {**document.meta, "extraction_error": result["error"]}
+            return replace(document, meta=new_meta), False
+
+        # remove potentially existing error metadata from previous runs
+        new_meta = {**document.meta}
+        new_meta.pop("extraction_error", None)
+
+        # process the LLM response considering the possible response formats
+        response_text = result["replies"][0].text
+        content, meta_updates, error = LLMDocumentContentExtractor._process_response(response_text)
+
+        if error:
+            new_meta["extraction_error"] = error
+            return replace(document, meta=new_meta), False
+
+        new_meta.update(meta_updates)
+        final_content = document.content if content is None else content
+        return replace(document, content=final_content, meta=new_meta), True
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Run extraction on image-based documents. One LLM call per document.
+
+        :param documents: A list of image-based documents to process. Each must have a valid file path in its metadata.
+        :returns:
+            A dictionary with "documents" (successfully processed) and "failed_documents" (with failure metadata).
+        """
+        if not documents:
+            return {"documents": [], "failed_documents": []}
+
+        self.warm_up()
+
+        image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            results = executor.map(self._run_on_thread, image_contents)
+
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            doc, success = self._process_llm_results(document, result)
+            if success:
+                successful_documents.append(doc)
+            else:
+                failed_documents.append(doc)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    async def run_async(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Asynchronously run extraction on image-based documents. One LLM call per document.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in an async code. LLM calls are made concurrently, bounded by `max_workers`.
+        If the chat generator only implements a synchronous `run` method, it is executed in a thread to avoid
+        blocking the event loop.
+
+        :param documents: A list of image-based documents to process. Each must have a valid file path in its metadata.
+        :returns:
+            A dictionary with "documents" (successfully processed) and "failed_documents" (with failure metadata).
+        """
+        if not documents:
+            return {"documents": [], "failed_documents": []}
+
+        await self.warm_up_async()
+
+        image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
+
+        # Run the LLM on each image content, bounding concurrency per task so max_workers is enforced.
+        sem = asyncio.Semaphore(max(1, self.max_workers))
+
+        async def _bounded_run(image_content: ImageContent | None) -> dict[str, Any]:
+            async with sem:
+                return await self._run_async(image_content)
+
+        results = await asyncio.gather(*[_bounded_run(image_content) for image_content in image_contents])
+
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            doc, success = self._process_llm_results(document, result)
+            if success:
+                successful_documents.append(doc)
+            else:
+                failed_documents.append(doc)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}

haystack/components/extractors/llm_metadata_extractor.py

@@ -0,0 +1,474 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import copy
+import json
+from asyncio import Semaphore, gather
+from collections.abc import Iterable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from typing import Any
+
+from jinja2 import meta
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace, expand_page_range
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLMMetadataExtractor:
+    """
+    Extracts metadata from documents using a Large Language Model (LLM).
+
+    The metadata is extracted by providing a prompt to an LLM that generates the metadata.
+
+    This component expects as input a list of documents and a prompt. The prompt must have exactly one variable, called
+    `document`, that points to a single document in the list of documents. So to access the content of the document,
+    you can use `{{ document.content }}` in the prompt.
+
+    The component will run the LLM on each document in the list and extract metadata from the document. The metadata
+    will be added to the document's metadata field. If the LLM fails to extract metadata from a document, the document
+    will be added to the `failed_documents` list. The failed documents will have the keys `metadata_extraction_error` and
+    `metadata_extraction_response` in their metadata. These documents can be re-run with another extractor to
+    extract metadata by using the `metadata_extraction_response` and `metadata_extraction_error` in the prompt.
+
+    ```python
+    from haystack import Document
+    from haystack.components.extractors.llm_metadata_extractor import LLMMetadataExtractor
+    from haystack.components.generators.chat import OpenAIChatGenerator
+
+    NER_PROMPT = '''
+    -Goal-
+    Given text and a list of entity types, identify all entities of those types from the text.
+
+    -Steps-
+    1. Identify all entities. For each identified entity, extract the following information:
+    - entity: Name of the entity
+    - entity_type: One of the following types: [organization, product, service, industry]
+    Format each entity as a JSON like: {"entity": <entity_name>, "entity_type": <entity_type>}
+
+    2. Return output in a single list with all the entities identified in steps 1.
+
+    -Examples-
+    ######################
+    Example 1:
+    entity_types: [organization, person, partnership, financial metric, product, service, industry, investment strategy, market trend]
+    text: Another area of strength is our co-brand issuance. Visa is the primary network partner for eight of the top
+    10 co-brand partnerships in the US today and we are pleased that Visa has finalized a multi-year extension of
+    our successful credit co-branded partnership with Alaska Airlines, a portfolio that benefits from a loyal customer
+    base and high cross-border usage.
+    We have also had significant co-brand momentum in CEMEA. First, we launched a new co-brand card in partnership
+    with Qatar Airways, British Airways and the National Bank of Kuwait. Second, we expanded our strong global
+    Marriott relationship to launch Qatar's first hospitality co-branded card with Qatar Islamic Bank. Across the
+    United Arab Emirates, we now have exclusive agreements with all the leading airlines marked by a recent
+    agreement with Emirates Skywards.
+    And we also signed an inaugural Airline co-brand agreement in Morocco with Royal Air Maroc. Now newer digital
+    issuers are equally
+    ------------------------
+    output:
+    {"entities": [{"entity": "Visa", "entity_type": "company"}, {"entity": "Alaska Airlines", "entity_type": "company"}, {"entity": "Qatar Airways", "entity_type": "company"}, {"entity": "British Airways", "entity_type": "company"}, {"entity": "National Bank of Kuwait", "entity_type": "company"}, {"entity": "Marriott", "entity_type": "company"}, {"entity": "Qatar Islamic Bank", "entity_type": "company"}, {"entity": "Emirates Skywards", "entity_type": "company"}, {"entity": "Royal Air Maroc", "entity_type": "company"}]}
+    #############################
+    -Real Data-
+    ######################
+    entity_types: [company, organization, person, country, product, service]
+    text: {{ document.content }}
+    ######################
+    output:
+    '''
+
+    docs = [
+        Document(content="deepset was founded in 2018 in Berlin, and is known for its Haystack framework"),
+        Document(content="Hugging Face is a company that was founded in New York, USA and is known for its Transformers library")
+    ]
+
+    chat_generator = OpenAIChatGenerator(
+        generation_kwargs={
+            "max_completion_tokens": 500,
+            "temperature": 0.0,
+            "seed": 0,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "entity_extraction",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "entities": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "entity": {"type": "string"},
+                                        "entity_type": {"type": "string"}
+                                    },
+                                    "required": ["entity", "entity_type"],
+                                    "additionalProperties": False
+                                }
+                            }
+                        },
+                        "required": ["entities"],
+                        "additionalProperties": False
+                    }
+                }
+            },
+        },
+        max_retries=1,
+        timeout=60.0,
+    )
+
+    extractor = LLMMetadataExtractor(
+        prompt=NER_PROMPT,
+        chat_generator=chat_generator,
+        expected_keys=["entities"],
+        raise_on_failure=False,
+    )
+
+    extractor.run(documents=docs)
+    # >> {'documents': [
+    #     Document(id=.., content: 'deepset was founded in 2018 in Berlin, and is known for its Haystack framework',
+    #     meta: {'entities': [{'entity': 'deepset', 'entity_type': 'company'}, {'entity': 'Berlin', 'entity_type': 'city'},
+    #           {'entity': 'Haystack', 'entity_type': 'product'}]}),
+    #     Document(id=.., content: 'Hugging Face is a company that was founded in New York, USA and is known for its Transformers library',
+    #     meta: {'entities': [
+    #             {'entity': 'Hugging Face', 'entity_type': 'company'}, {'entity': 'New York', 'entity_type': 'city'},
+    #             {'entity': 'USA', 'entity_type': 'country'}, {'entity': 'Transformers', 'entity_type': 'product'}
+    #             ]})
+    #        ]
+    #     'failed_documents': []
+    #    }
+    # >>
+    ```
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        prompt: str,
+        chat_generator: ChatGenerator,
+        expected_keys: list[str] | None = None,
+        page_range: list[str | int] | None = None,
+        raise_on_failure: bool = False,
+        max_workers: int = 3,
+    ) -> None:
+        """
+        Initializes the LLMMetadataExtractor.
+
+        :param prompt: The prompt to be used for the LLM. It must contain exactly one variable, called `document`,
+            which points to a single document in the list of documents. For example, to access the content of the
+            document, use `{{ document.content }}` in the prompt.
+        :param chat_generator: a ChatGenerator instance which represents the LLM. In order for the component to work,
+            the LLM should be configured to return a JSON object. For example, when using the OpenAIChatGenerator, you
+            should pass `{"response_format": {"type": "json_object"}}` in the `generation_kwargs`.
+        :param expected_keys: The keys expected in the JSON output from the LLM.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+            metadata from the first and third pages of each document. It also accepts printable range strings, e.g.:
+            ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,11, 12.
+            If None, metadata will be extracted from the entire document for each document in the documents list.
+            This parameter is optional and can be overridden in the `run` method.
+        :param raise_on_failure: Whether to raise an error on failure during the execution of the Generator or
+            validation of the JSON output.
+        :param max_workers: The maximum number of workers to use in the thread pool executor.
+            This parameter is used limit the maximum number of requests that should be allowed to run concurrently
+            when using the `run_async` method.
+        """
+        self.prompt = prompt
+        ast = SandboxedEnvironment().parse(prompt)
+        template_variables = meta.find_undeclared_variables(ast)
+        variables = list(template_variables)
+        if variables != ["document"]:
+            raise ValueError(
+                f"Prompt must have exactly one variable called 'document'. "
+                f"Found {','.join(variables) or 'no variables'} in the prompt."
+            )
+        self.builder = PromptBuilder(prompt, required_variables=variables)
+        self.raise_on_failure = raise_on_failure
+        self.expected_keys = expected_keys or []
+        self.splitter = DocumentSplitter(split_by="page", split_length=1)
+        self.expanded_range = expand_page_range(page_range) if page_range else None
+        self.max_workers = max_workers
+        self._chat_generator = chat_generator
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator and splitter.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "warm_up"):
+                inner.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator and splitter on the serving event loop.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "warm_up_async"):
+                await inner.warm_up_async()
+            elif hasattr(inner, "warm_up"):
+                inner.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's and splitter's resources.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "close"):
+                inner.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's and splitter's async resources.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "close_async"):
+                await inner.close_async()
+            elif hasattr(inner, "close"):
+                inner.close()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+
+        return default_to_dict(
+            self,
+            prompt=self.prompt,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            expected_keys=self.expected_keys,
+            page_range=self.expanded_range,
+            raise_on_failure=self.raise_on_failure,
+            max_workers=self.max_workers,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMMetadataExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary with serialized data.
+        :returns:
+            An instance of the component.
+        """
+
+        deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)
+
+    def _extract_metadata(self, llm_answer: str) -> dict[str, Any]:
+        try:
+            parsed_metadata = _parse_dict_from_json(llm_answer, expected_keys=self.expected_keys, raise_on_failure=True)
+        except (ValueError, json.JSONDecodeError) as e:
+            logger.warning(
+                "Response from the LLM is not valid JSON or missing expected keys. Received output: {response}",
+                response=llm_answer,
+            )
+            if self.raise_on_failure:
+                raise e
+            return {"error": "Response is not valid JSON or missing keys. Error: " + str(e)}
+
+        return parsed_metadata
+
+    def _prepare_prompts(
+        self, documents: list[Document], expanded_range: list[int] | None = None
+    ) -> list[ChatMessage | None]:
+        all_prompts: list[ChatMessage | None] = []
+        for document in documents:
+            if not document.content:
+                logger.warning("Document {doc_id} has no content. Skipping metadata extraction.", doc_id=document.id)
+                all_prompts.append(None)
+                continue
+
+            if expanded_range:
+                doc_copy = copy.deepcopy(document)
+                pages = self.splitter.run(documents=[doc_copy])
+                content = ""
+                for idx, page in enumerate(pages["documents"]):
+                    if idx + 1 in expanded_range and page.content is not None:
+                        content += page.content
+                doc_copy = replace(doc_copy, content=content)
+            else:
+                doc_copy = document
+
+            prompt_with_doc = self.builder.run(template=self.prompt, template_variables={"document": doc_copy})
+
+            # build a ChatMessage with the prompt
+            message = ChatMessage.from_user(prompt_with_doc["prompt"])
+            all_prompts.append(message)
+
+        return all_prompts
+
+    def _run_on_thread(self, prompt: ChatMessage | None) -> dict[str, Any]:
+        # If prompt is None, return an error dictionary
+        if prompt is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        try:
+            result = self._chat_generator.run(messages=[prompt])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+        return result
+
+    async def _run_async(self, prompt: ChatMessage | None) -> dict[str, Any]:
+        # If prompt is None, return an error dictionary
+        if prompt is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        try:
+            result = await _execute_component_async(self._chat_generator, messages=[prompt])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+        return result
+
+    def _process_results(
+        self, documents: list[Document], results: Iterable[dict[str, Any]]
+    ) -> tuple[list[Document], list[Document]]:
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            new_meta = {**document.meta}
+            if "error" in result:
+                new_meta["metadata_extraction_error"] = result["error"]
+                new_meta["metadata_extraction_response"] = None
+                failed_documents.append(replace(document, meta=new_meta))
+                continue
+
+            parsed_metadata = self._extract_metadata(result["replies"][0].text)
+            if "error" in parsed_metadata:
+                new_meta["metadata_extraction_error"] = parsed_metadata["error"]
+                new_meta["metadata_extraction_response"] = result["replies"][0]
+                failed_documents.append(replace(document, meta=new_meta))
+                continue
+
+            for key in parsed_metadata:
+                new_meta[key] = parsed_metadata[key]
+                # Remove metadata_extraction_error and metadata_extraction_response if present from previous runs
+                new_meta.pop("metadata_extraction_error", None)
+                new_meta.pop("metadata_extraction_response", None)
+            successful_documents.append(replace(document, meta=new_meta))
+        return successful_documents, failed_documents
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    def run(self, documents: list[Document], page_range: list[str | int] | None = None) -> dict[str, Any]:
+        """
+        Extract metadata from documents using a Large Language Model.
+
+        If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
+        will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
+        extracted from the entire document if `page_range` is not provided.
+
+        The original documents will be returned  updated with the extracted metadata.
+
+        :param documents: List of documents to extract metadata from.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+                           metadata from the first and third pages of each document. It also accepts printable range
+                           strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
+                           11, 12.
+                           If None, metadata will be extracted from the entire document for each document in the
+                           documents list.
+        :returns:
+            A dictionary with the keys:
+            - "documents": A list of documents that were successfully updated with the extracted metadata.
+            - "failed_documents": A list of documents that failed to extract metadata. These documents will have
+            "metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
+            re-run with the extractor to extract metadata.
+        """
+        if len(documents) == 0:
+            logger.warning("No documents provided. Skipping metadata extraction.")
+            return {"documents": [], "failed_documents": []}
+
+        self.warm_up()
+
+        expanded_range = self.expanded_range
+        if page_range:
+            expanded_range = expand_page_range(page_range)
+
+        # Create ChatMessage prompts for each document
+        all_prompts = self._prepare_prompts(documents=documents, expanded_range=expanded_range)
+
+        # Run the LLM on each prompt
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            results = executor.map(self._run_on_thread, all_prompts)
+
+        successful_documents, failed_documents = self._process_results(documents, results)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    async def run_async(self, documents: list[Document], page_range: list[str | int] | None = None) -> dict[str, Any]:
+        """
+        Asynchronously extract metadata from documents using a Large Language Model.
+
+        If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
+        will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
+        extracted from the entire document if `page_range` is not provided.
+
+        The original documents will be returned  updated with the extracted metadata.
+
+        This is the asynchronous version of the `run` method. It has the same parameters
+        and return values but can be used with `await` in an async code.
+
+        :param documents: List of documents to extract metadata from.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+                           metadata from the first and third pages of each document. It also accepts printable range
+                           strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
+                           11, 12.
+                           If None, metadata will be extracted from the entire document for each document in the
+                           documents list.
+        :returns:
+            A dictionary with the keys:
+            - "documents": A list of documents that were successfully updated with the extracted metadata.
+            - "failed_documents": A list of documents that failed to extract metadata. These documents will have
+            "metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
+            re-run with the extractor to extract metadata.
+        """
+        if len(documents) == 0:
+            logger.warning("No documents provided. Skipping metadata extraction.")
+            return {"documents": [], "failed_documents": []}
+
+        await self.warm_up_async()
+
+        expanded_range = self.expanded_range
+        if page_range:
+            expanded_range = expand_page_range(page_range)
+
+        # Create ChatMessage prompts for each document
+        all_prompts = self._prepare_prompts(documents=documents, expanded_range=expanded_range)
+
+        # Run the LLM on each prompt, bounding concurrency per task so max_workers is enforced.
+        sem = Semaphore(max(1, self.max_workers))
+
+        async def _bounded_run(prompt: ChatMessage | None) -> dict[str, Any]:
+            async with sem:
+                return await self._run_async(prompt)
+
+        results = await gather(*[_bounded_run(prompt) for prompt in all_prompts])
+
+        successful_documents, failed_documents = self._process_results(documents, results)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}

haystack/components/extractors/regex_text_extractor.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from typing import Any
+
+from haystack import component, logging
+from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class RegexTextExtractor:
+    """
+    Extracts text from chat message or string input using a regex pattern.
+
+    RegexTextExtractor parses input text or ChatMessages using a provided regular expression pattern.
+    It can be configured to search through all messages or only the last message in a list of ChatMessages.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.extractors import RegexTextExtractor
+    from haystack.dataclasses import ChatMessage
+
+    # Using with a string
+    parser = RegexTextExtractor(regex_pattern='<issue url=\"(.+)\">')
+    result = parser.run(text_or_messages='<issue url="github.com/hahahaha">hahahah</issue>')
+    # result: {"captured_text": "github.com/hahahaha"}
+
+    # Using with ChatMessages
+    messages = [ChatMessage.from_user('<issue url="github.com/hahahaha">hahahah</issue>')]
+    result = parser.run(text_or_messages=messages)
+    # result: {"captured_text": "github.com/hahahaha"}
+    ```
+    """
+
+    def __init__(self, regex_pattern: str) -> None:
+        """
+        Creates an instance of the RegexTextExtractor component.
+
+        :param regex_pattern:
+            The regular expression pattern used to extract text.
+            The pattern should include a capture group to extract the desired text.
+            Example: `'<issue url="(.+)">'` captures `'github.com/hahahaha'` from `'<issue url="github.com/hahahaha">'`.
+        """
+        self.regex_pattern = regex_pattern
+
+        # Check if the pattern has at least one capture group
+        num_groups = re.compile(regex_pattern).groups
+        if num_groups < 1:
+            logger.warning(
+                "The provided regex pattern {regex_pattern} doesn't contain any capture groups. "
+                "The entire match will be returned instead.",
+                regex_pattern=regex_pattern,
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, regex_pattern=self.regex_pattern)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "RegexTextExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        # return_empty_on_no_match is an old parameter. We'd like to avoid that pipelines break if it's still present.
+        if "return_empty_on_no_match" in data["init_parameters"]:
+            logger.warning("The `return_empty_on_no_match` init parameter has been removed and will be ignored.")
+            data["init_parameters"].pop("return_empty_on_no_match")
+
+        return default_from_dict(cls, data)
+
+    @component.output_types(captured_text=str)
+    def run(self, text_or_messages: str | list[ChatMessage]) -> dict[str, str]:
+        """
+        Extracts text from input using the configured regex pattern.
+
+        :param text_or_messages:
+            Either a string or a list of ChatMessage objects to search through.
+
+        :returns:
+          - `{"captured_text": "matched text"}` if a match is found
+          - `{"captured_text": ""}` if no match is found
+
+        :raises TypeError: if receiving a list the last element is not a ChatMessage instance.
+        """
+        if isinstance(text_or_messages, str):
+            return self._build_result(self._extract_from_text(text_or_messages))
+        if not text_or_messages:
+            logger.warning("Received empty list of messages")
+            return {"captured_text": ""}
+        return self._process_last_message(text_or_messages)
+
+    def _build_result(self, result: str | list[str]) -> dict:
+        """Helper method to build the return dictionary based on configuration."""
+        if (isinstance(result, str) and result == "") or (isinstance(result, list) and not result):
+            return {"captured_text": ""}
+        return {"captured_text": result}
+
+    def _process_last_message(self, messages: list[ChatMessage]) -> dict:
+        """
+        Process only the last message and build the result.
+
+        :raises TypeError: If the last element of the list is not a ChatMessage instance.
+        """
+        last_message = messages[-1]
+        if not isinstance(last_message, ChatMessage):
+            raise TypeError(f"Expected ChatMessage object, got {type(last_message)}")
+        if last_message.text is None:
+            logger.warning("Last message has no text content")
+            return {"captured_text": ""}
+        result = self._extract_from_text(last_message.text)
+        return self._build_result(result)
+
+    def _extract_from_text(self, text: str) -> str | list[str]:
+        """
+        Extract text using the regex pattern.
+
+        :param text:
+            The text to search through.
+
+        :returns:
+            The text captured by the first capturing group in the regex pattern.
+            If the pattern has no capture groups, returns the entire match.
+            If no match is found, returns an empty string.
+        """
+        match = re.search(self.regex_pattern, text)
+        if not match:
+            return ""
+        if match.groups():
+            return match.group(1)
+        return match.group(0)

haystack/components/fetchers/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"link_content": ["LinkContentFetcher"]}
+
+if TYPE_CHECKING:
+    from .link_content import LinkContentFetcher as LinkContentFetcher
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/fetchers/link_content.py

@@ -0,0 +1,493 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+from collections import defaultdict
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from fnmatch import fnmatch
+from typing import Any, cast
+
+import httpx
+from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from haystack import component, logging
+from haystack.dataclasses import ByteStream
+from haystack.lazy_imports import LazyImport
+from haystack.version import __version__
+
+# HTTP/2 support via lazy import
+with LazyImport("Run 'pip install httpx[http2]' to use HTTP/2 support") as h2_import:
+    pass  # nothing to import as we simply set the http2 attribute, library handles the rest
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_USER_AGENT = f"haystack/LinkContentFetcher/{__version__}"
+
+REQUEST_HEADERS = {
+    "accept": "*/*",
+    "User-Agent": DEFAULT_USER_AGENT,
+    "Accept-Language": "en-US,en;q=0.9,it;q=0.8,es;q=0.7",
+    "referer": "https://www.google.com/",
+}
+
+
+def _merge_headers(*args: dict[str, str]) -> dict[str, str]:
+    """
+    Merge a list of dict using case-insensitively
+
+    :param args: a list of dict to merge
+    :returns: The merged dict
+    """
+    merged = {}
+    keymap = {}
+
+    for d in args:
+        for k, v in d.items():
+            kl = k.lower()
+            keymap[kl] = k
+            merged[kl] = v
+
+    return {keymap[kl]: v for kl, v in merged.items()}
+
+
+def _text_content_handler(response: httpx.Response) -> ByteStream:
+    """
+    Handles text content.
+
+    :param response: Response object from the request.
+    :returns: The extracted text.
+    """
+    return ByteStream.from_string(response.text)
+
+
+def _binary_content_handler(response: httpx.Response) -> ByteStream:
+    """
+    Handles binary content.
+
+    :param response: Response object from the request.
+    :returns: The extracted binary file-like object.
+    """
+    return ByteStream(data=response.content)
+
+
+@component
+class LinkContentFetcher:
+    """
+    Fetches and extracts content from URLs.
+
+    It supports various content types, retries on failures, and automatic user-agent rotation for failed web
+    requests. Use it as the data-fetching step in your pipelines.
+
+    You may need to convert LinkContentFetcher's output into a list of documents. Use HTMLToDocument
+    converter to do this.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.fetchers.link_content import LinkContentFetcher
+
+    fetcher = LinkContentFetcher()
+    streams = fetcher.run(urls=["https://www.google.com"])["streams"]
+
+    assert len(streams) == 1
+    assert streams[0].meta == {'content_type': 'text/html', 'url': 'https://www.google.com'}
+    assert streams[0].data
+    ```
+
+    For async usage:
+
+    ```python
+    import asyncio
+    from haystack.components.fetchers import LinkContentFetcher
+
+    async def fetch_async():
+        fetcher = LinkContentFetcher()
+        result = await fetcher.run_async(urls=["https://www.google.com"])
+        return result["streams"]
+
+    streams = asyncio.run(fetch_async())
+    ```
+    """
+
+    def __init__(
+        self,
+        raise_on_failure: bool = True,
+        user_agents: list[str] | None = None,
+        retry_attempts: int = 2,
+        timeout: int = 3,
+        http2: bool = False,
+        client_kwargs: dict | None = None,
+        request_headers: dict[str, str] | None = None,
+    ) -> None:
+        """
+        Initializes the component.
+
+        :param raise_on_failure: If `True`, raises an exception if it fails to fetch a single URL.
+            For multiple URLs, it logs errors and returns the content it successfully fetched.
+        :param user_agents: [User agents](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent)
+            for fetching content. If `None`, a default user agent is used.
+        :param retry_attempts: The number of times to retry to fetch the URL's content.
+        :param timeout: Timeout in seconds for the request.
+        :param http2: Whether to enable HTTP/2 support for requests. Defaults to False.
+                     Requires the 'h2' package to be installed (via `pip install httpx[http2]`).
+        :param client_kwargs: Additional keyword arguments to pass to the httpx client.
+                     If `None`, default values are used.
+        """
+        self.raise_on_failure = raise_on_failure
+        self.user_agents = user_agents or [DEFAULT_USER_AGENT]
+        self.current_user_agent_idx: int = 0
+        self.retry_attempts = retry_attempts
+        self.timeout = timeout
+        self.http2 = http2
+        self.client_kwargs = client_kwargs or {}
+        self.request_headers = request_headers or {}
+
+        # Configure default client settings
+        self.client_kwargs.setdefault("timeout", timeout)
+        self.client_kwargs.setdefault("follow_redirects", True)
+
+        # httpx clients are built lazily in warm_up / warm_up_async (resource lifecycle)
+        self._client: httpx.Client | None = None
+        self._async_client: httpx.AsyncClient | None = None
+
+        # register default content handlers that extract data from the response
+        self.handlers: dict[str, Callable[[httpx.Response], ByteStream]] = defaultdict(lambda: _text_content_handler)
+        self.handlers["text/*"] = _text_content_handler
+        self.handlers["text/html"] = _binary_content_handler
+        self.handlers["application/json"] = _text_content_handler
+        self.handlers["application/*"] = _binary_content_handler
+        self.handlers["image/*"] = _binary_content_handler
+        self.handlers["audio/*"] = _binary_content_handler
+        self.handlers["video/*"] = _binary_content_handler
+
+        @retry(
+            reraise=True,
+            stop=stop_after_attempt(self.retry_attempts),
+            wait=wait_exponential(multiplier=1, min=2, max=10),
+            retry=(retry_if_exception_type((httpx.HTTPStatusError, httpx.RequestError))),
+            # This method is invoked only after failed requests (exception raised)
+            after=self._switch_user_agent,
+        )
+        def get_response(url: str) -> httpx.Response:
+            assert self._client is not None  # mypy: client is built by warm_up before run
+            response = self._client.get(url, headers=self._get_headers())
+            response.raise_for_status()
+            return response
+
+        self._get_response: Callable = get_response
+
+    def _build_client_kwargs(self) -> dict[str, Any]:
+        """
+        Build the keyword arguments used to construct the httpx clients.
+
+        Resolves optional HTTP/2 support, downgrading to HTTP/1.1 if the 'h2' package is not installed.
+        """
+        client_kwargs = {**self.client_kwargs}
+
+        # Optional HTTP/2 support
+        if self.http2:
+            try:
+                h2_import.check()
+                client_kwargs["http2"] = True
+            except ImportError:
+                logger.warning(
+                    "HTTP/2 support requested but 'h2' package is not installed. "
+                    "Falling back to HTTP/1.1. Install with `pip install httpx[http2]` to enable HTTP/2 support."
+                )
+                self.http2 = False  # Update the setting to match actual capability
+
+        return client_kwargs
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous httpx client.
+        """
+        if self._client is None:
+            self._client = httpx.Client(**self._build_client_kwargs())
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous httpx client on the serving event loop.
+        """
+        if self._async_client is None:
+            self._async_client = httpx.AsyncClient(**self._build_client_kwargs())
+
+    def close(self) -> None:
+        """
+        Releases the synchronous httpx client.
+        """
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous httpx client.
+        """
+        if self._async_client is not None:
+            await self._async_client.aclose()
+            self._async_client = None
+
+    def _get_headers(self) -> dict[str, str]:
+        """
+        Build headers with precedence
+
+        client defaults -> component defaults -> user-provided -> rotating UA
+        """
+        base = dict(self._client.headers) if self._client is not None else {}
+        return _merge_headers(
+            base, REQUEST_HEADERS, self.request_headers, {"User-Agent": self.user_agents[self.current_user_agent_idx]}
+        )
+
+    @component.output_types(streams=list[ByteStream])
+    def run(self, urls: list[str]) -> dict[str, Any]:
+        """
+        Fetches content from a list of URLs and returns a list of extracted content streams.
+
+        Each content stream is a `ByteStream` object containing the extracted content as binary data.
+        Each ByteStream object in the returned list corresponds to the contents of a single URL.
+        The content type of each stream is stored in the metadata of the ByteStream object under
+        the key "content_type". The URL of the fetched content is stored under the key "url".
+
+        :param urls: A list of URLs to fetch content from.
+        :returns: `ByteStream` objects representing the extracted content.
+
+        :raises Exception: If the provided list of URLs contains only a single URL, and `raise_on_failure` is set to
+            `True`, an exception will be raised in case of an error during content retrieval.
+            In all other scenarios, any retrieval errors are logged, and a list of successfully retrieved `ByteStream`
+             objects is returned.
+        """
+        self.warm_up()
+
+        streams: list[ByteStream] = []
+        if not urls:
+            return {"streams": streams}
+
+        # don't use multithreading if there's only one URL
+        if len(urls) == 1:
+            stream_metadata, stream = self._fetch(urls[0])
+            stream.meta.update(stream_metadata)
+            stream = replace(stream, mime_type=stream.meta.get("content_type", None))
+            streams.append(stream)
+        else:
+            with ThreadPoolExecutor() as executor:
+                results = executor.map(self._fetch_with_exception_suppression, urls)
+
+            for stream_metadata, stream in results:  # type: ignore
+                if stream_metadata is not None and stream is not None:
+                    stream.meta.update(stream_metadata)
+                    stream = replace(stream, mime_type=stream.meta.get("content_type", None))
+                    streams.append(stream)
+
+        return {"streams": streams}
+
+    @component.output_types(streams=list[ByteStream])
+    async def run_async(self, urls: list[str]) -> dict[str, Any]:
+        """
+        Asynchronously fetches content from a list of URLs and returns a list of extracted content streams.
+
+        This is the asynchronous version of the `run` method with the same parameters and return values.
+
+        :param urls: A list of URLs to fetch content from.
+        :returns: `ByteStream` objects representing the extracted content.
+        """
+        await self.warm_up_async()
+
+        streams: list[ByteStream] = []
+        if not urls:
+            return {"streams": streams}
+
+        assert self._async_client is not None  # mypy: async_client is built by warm_up_async above
+        # Create tasks for all URLs using _fetch_async directly
+        tasks = [self._fetch_async(url, self._async_client) for url in urls]
+
+        # Only capture exceptions when we have multiple URLs or raise_on_failure=False
+        # This ensures errors propagate appropriately for single URLs with raise_on_failure=True
+        return_exceptions = not (len(urls) == 1 and self.raise_on_failure)
+        results = await asyncio.gather(*tasks, return_exceptions=return_exceptions)
+
+        # Process results
+        for i, result in enumerate(results):
+            # Handle exception results (only happens when return_exceptions=True)
+            if isinstance(result, Exception):
+                logger.warning("Error fetching {url}: {error}", url=urls[i], error=str(result))
+                # Add an empty result for failed URLs when raise_on_failure=False
+                if not self.raise_on_failure:
+                    streams.append(ByteStream(data=b"", meta={"content_type": "Unknown", "url": urls[i]}))
+                continue
+
+            # Process successful results
+            # At this point, result is not an exception, so we need to cast it to the correct type for mypy
+            if not isinstance(result, Exception):  # Runtime check
+                # Use cast to tell mypy that result is the tuple type returned by _fetch_async
+                result_tuple = cast(tuple[dict[str, str] | None, ByteStream | None], result)
+                stream_metadata, stream = result_tuple
+                if stream_metadata is not None and stream is not None:
+                    stream.meta.update(stream_metadata)
+                    stream = replace(stream, mime_type=stream.meta.get("content_type", None))
+                    streams.append(stream)
+
+        return {"streams": streams}
+
+    def _fetch(self, url: str) -> tuple[dict[str, str], ByteStream]:
+        """
+        Fetches content from a URL and returns it as a ByteStream.
+
+        :param url: The URL to fetch content from.
+        :returns: A tuple containing the ByteStream metadata dict and the corresponding ByteStream.
+             ByteStream metadata contains the URL and the content type of the fetched content.
+             The content type is a string indicating the type of content fetched (for example, "text/html",
+             "application/pdf"). The ByteStream object contains the fetched content as binary data.
+
+        :raises: If an error occurs during content retrieval and `raise_on_failure` is set to True, this method will
+        raise an exception. Otherwise, all fetching errors are logged, and an empty ByteStream is returned.
+
+        """
+        content_type: str = "text/html"
+        stream: ByteStream = ByteStream(data=b"")
+        try:
+            response = self._get_response(url)
+            content_type = self._get_content_type(response)
+            handler: Callable = self._resolve_handler(content_type)
+            stream = handler(response)
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            # less verbose log as this is expected to happen often (requests failing, blocked, etc.)
+            logger.debug("Couldn't retrieve content from {url} because {error}", url=url, error=str(e))
+
+        finally:
+            self.current_user_agent_idx = 0
+
+        return {"content_type": content_type, "url": url}, stream
+
+    async def _fetch_async(
+        self, url: str, client: httpx.AsyncClient
+    ) -> tuple[dict[str, str] | None, ByteStream | None]:
+        """
+        Asynchronously fetches content from a URL and returns it as a ByteStream.
+
+        :param url: The URL to fetch content from.
+        :param client: The async httpx client to use for making requests.
+        :returns: A tuple containing the ByteStream metadata dict and the corresponding ByteStream.
+        """
+        content_type: str = "text/html"
+        stream: ByteStream | None = None
+        metadata: dict[str, str] | None = None
+
+        try:
+            response = await self._get_response_async(url, client)
+            content_type = self._get_content_type(response)
+            handler: Callable = self._resolve_handler(content_type)
+            stream = handler(response)
+            metadata = {"content_type": content_type, "url": url}
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.debug("Couldn't retrieve content from {url} because {error}", url=url, error=str(e))
+            # Create an empty ByteStream for failed requests when raise_on_failure is False
+            stream = ByteStream(data=b"")
+            metadata = {"content_type": content_type, "url": url}
+        finally:
+            self.current_user_agent_idx = 0
+
+        return metadata, stream
+
+    def _fetch_with_exception_suppression(self, url: str) -> tuple[dict[str, str] | None, ByteStream | None]:
+        """
+        Fetches content from a URL and returns it as a ByteStream.
+
+        If `raise_on_failure` is set to True, this method will wrap the fetch() method and catch any exceptions.
+        Otherwise, it will simply call the fetch() method.
+        :param url: The URL to fetch content from.
+        :returns: A tuple containing the ByteStream metadata dict and the corresponding ByteStream.
+
+        """
+        if self.raise_on_failure:
+            try:
+                return self._fetch(url)
+            except Exception as e:
+                logger.warning("Error fetching {url}: {error}", url=url, error=str(e))
+                return {"content_type": "Unknown", "url": url}, None
+        else:
+            return self._fetch(url)
+
+    async def _get_response_async(self, url: str, client: httpx.AsyncClient) -> httpx.Response:
+        """
+        Asynchronously gets a response from a URL with retry logic.
+
+        :param url: The URL to fetch.
+        :param client: The async httpx client to use for making requests.
+        :returns: The httpx Response object.
+        """
+        attempt = 0
+        last_exception = None
+
+        while attempt <= self.retry_attempts:
+            try:
+                response = await client.get(url, headers=self._get_headers())
+                response.raise_for_status()
+                return response
+            except (httpx.HTTPStatusError, httpx.RequestError) as e:
+                last_exception = e
+                attempt += 1
+                if attempt <= self.retry_attempts:
+                    self._switch_user_agent(None)  # Switch user agent for next retry
+                    # Wait before retry using exponential backoff
+                    await asyncio.sleep(min(2 * 2 ** (attempt - 1), 10))
+                else:
+                    break
+
+        # If we've exhausted all retries, raise the last exception
+        if last_exception:
+            raise last_exception
+
+        # This should never happen, but just in case
+        raise httpx.RequestError("Failed to get response after retries", request=None)
+
+    def _get_content_type(self, response: httpx.Response) -> str:
+        """
+        Get the content type of the response.
+
+        :param response: The response object.
+        :returns: The content type of the response.
+        """
+        content_type = response.headers.get("Content-Type", "")
+        return content_type.split(";")[0]
+
+    def _resolve_handler(self, content_type: str) -> Callable[[httpx.Response], ByteStream]:
+        """
+        Resolves the handler for the given content type.
+
+        First, it tries to find a direct match for the content type in the handlers dictionary.
+        If no direct match is found, it tries to find a pattern match using the fnmatch function.
+        If no pattern match is found, it returns the default handler for text/plain.
+
+        :param content_type: The content type to resolve the handler for.
+        :returns: The handler for the given content type, if found. Otherwise, the default handler for text/plain.
+        """
+        # direct match
+        if content_type in self.handlers:
+            return self.handlers[content_type]
+
+        # pattern matches
+        for pattern, handler in self.handlers.items():
+            if fnmatch(content_type, pattern):
+                return handler
+
+        # default handler
+        return self.handlers["text/plain"]
+
+    def _switch_user_agent(self, retry_state: RetryCallState | None = None) -> None:  # noqa: ARG002
+        """
+        Switches the User-Agent for this LinkContentRetriever to the next one in the list of user agents.
+
+        Used by tenacity to retry the requests with a different user agent.
+
+        :param retry_state: The retry state (unused, required by tenacity).
+        """
+        self.current_user_agent_idx = (self.current_user_agent_idx + 1) % len(self.user_agents)
+        logger.debug("Switched user agent to {user_agent}", user_agent=self.user_agents[self.current_user_agent_idx])

haystack/components/generators/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"openai_image_generator": ["OpenAIImageGenerator"]}
+
+if TYPE_CHECKING:
+    from .openai_image_generator import OpenAIImageGenerator as OpenAIImageGenerator
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/generators/chat/__init__.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "openai": ["OpenAIChatGenerator"],
+    "openai_responses": ["OpenAIResponsesChatGenerator"],
+    "azure": ["AzureOpenAIChatGenerator"],
+    "azure_responses": ["AzureOpenAIResponsesChatGenerator"],
+    "fallback": ["FallbackChatGenerator"],
+    "llm": ["LLM"],
+    "mock": ["MockChatGenerator"],
+}
+
+if TYPE_CHECKING:
+    from .azure import AzureOpenAIChatGenerator as AzureOpenAIChatGenerator
+    from .azure_responses import AzureOpenAIResponsesChatGenerator as AzureOpenAIResponsesChatGenerator
+    from .fallback import FallbackChatGenerator as FallbackChatGenerator
+    from .llm import LLM as LLM
+    from .mock import MockChatGenerator as MockChatGenerator
+    from .openai import OpenAIChatGenerator as OpenAIChatGenerator
+    from .openai_responses import OpenAIResponsesChatGenerator as OpenAIResponsesChatGenerator
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/generators/chat/azure.py

@@ -0,0 +1,369 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any, ClassVar
+
+from openai.lib._pydantic import to_strict_json_schema
+from openai.lib.azure import AsyncAzureADTokenProvider, AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.dataclasses.streaming_chunk import StreamingCallbackT
+from haystack.tools import (
+    ToolsType,
+    _check_duplicate_tool_names,
+    deserialize_tools_or_toolset_inplace,
+    flatten_tools_or_toolsets,
+    serialize_tools_or_toolset,
+    warm_up_tools,
+)
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+
+@component
+class AzureOpenAIChatGenerator(OpenAIChatGenerator):
+    """
+    Generates text using OpenAI's models on Azure.
+
+    It works with the gpt-4 - type models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.ChatCompletion.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.generators.chat import AzureOpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+    from haystack.utils import Secret
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = AzureOpenAIChatGenerator(
+        azure_endpoint="<Your Azure endpoint e.g. `https://your-company.azure.openai.com/>",
+        api_key=Secret.from_token("<your-api-key>"),
+        azure_deployment="<this is a model name, e.g. gpt-4.1-mini>")
+    response = client.run(messages)
+    print(response)
+    ```
+
+    ```
+    {'replies':
+        [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
+        "Natural Language Processing (NLP) is a branch of artificial intelligence that focuses on
+         enabling computers to understand, interpret, and generate human language in a way that is useful.")],
+         _name=None,
+         _meta={'model': 'gpt-4.1-mini', 'index': 0, 'finish_reason': 'stop',
+         'usage': {'prompt_tokens': 15, 'completion_tokens': 36, 'total_tokens': 51}})]
+    }
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5.4",
+        "gpt-5.4-pro",
+        "gpt-5.3-codex",
+        "gpt-5.2",
+        "gpt-5.2-codex",
+        "gpt-5.2-chat",
+        "gpt-5.1",
+        "gpt-5.1-chat",
+        "gpt-5.1-codex",
+        "gpt-5.1-codex-mini",
+        "gpt-5",
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5-chat",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-4.1-nano",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4o-audio-preview",
+        "gpt-realtime-1.5",
+        "gpt-audio-1.5",
+        "o1",
+        "o1-mini",
+        "o3",
+        "o3-mini",
+        "o4-mini",
+        "codex-mini",
+        "gpt-4",
+        "gpt-35-turbo",
+        "gpt-oss-120b",
+        "computer-use-preview",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure
+    for the full list."""
+
+    # ruff: noqa: PLR0913
+    def __init__(
+        self,
+        azure_endpoint: str | Secret | None = None,
+        api_version: str | Secret | None = "2024-12-01-preview",
+        azure_deployment: str | None = "gpt-4.1-mini",
+        api_key: Secret | None = Secret.from_env_var("AZURE_OPENAI_API_KEY", strict=False),
+        azure_ad_token: Secret | None = Secret.from_env_var("AZURE_OPENAI_AD_TOKEN", strict=False),
+        organization: str | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        default_headers: dict[str, str] | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        *,
+        azure_ad_token_provider: AzureADTokenProvider | AsyncAzureADTokenProvider | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initialize the Azure OpenAI Chat Generator component.
+
+        :param azure_endpoint: The endpoint of the deployed model, for example `"https://example-resource.azure.openai.com/"`.
+            Can also be a [Secret](https://docs.haystack.deepset.ai/docs/secret-management), for example
+            `Secret.from_env_var("AZURE_OPENAI_ENDPOINT")`, to resolve the value from an environment variable at
+            runtime. This is useful to switch endpoints between environments (e.g. dev and prod) without changing the
+            serialized pipeline.
+        :param api_version: The version of the API to use. Defaults to 2024-12-01-preview.
+            Can also be a [Secret](https://docs.haystack.deepset.ai/docs/secret-management), for example
+            `Secret.from_env_var("AZURE_OPENAI_API_VERSION")`, to resolve the value from an environment variable at
+            runtime.
+        :param azure_deployment: The deployment of the model, usually the model name.
+        :param api_key: The API key to use for authentication.
+        :param azure_ad_token: [Azure Active Directory token](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id).
+        :param organization: Your organization ID, defaults to `None`. For help, see
+        [Setting up your organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param streaming_callback: A callback function called when a new token is received from the stream.
+            It accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param timeout: Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent directly to
+            the OpenAI endpoint. For details, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+            Some of the supported parameters:
+            - `max_completion_tokens`: An upper bound for the number of tokens that can be generated for a completion,
+                including visible output tokens and reasoning tokens.
+            - `temperature`: The sampling temperature to use. Higher values mean the model takes more risks.
+                Try 0.9 for more creative applications and 0 (argmax sampling) for ones with a well-defined answer.
+            - `top_p`: Nucleus sampling is an alternative to sampling with temperature, where the model considers
+                tokens with a top_p probability mass. For example, 0.1 means only the tokens comprising
+                the top 10% probability mass are considered.
+            - `n`: The number of completions to generate for each prompt. For example, with 3 prompts and n=2,
+                the LLM will generate two completions per prompt, resulting in 6 completions total.
+            - `stop`: One or more sequences after which the LLM should stop generating tokens.
+            - `presence_penalty`: The penalty applied if a token is already present.
+                Higher values make the model less likely to repeat the token.
+            - `frequency_penalty`: Penalty applied if a token has already been generated.
+                Higher values make the model less likely to repeat the token.
+            - `logit_bias`: Adds a logit bias to specific tokens. The keys of the dictionary are tokens, and the
+                values are the bias to add to that token.
+            - `response_format`: A JSON schema or a Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+                Notes:
+                - This parameter accepts Pydantic models and JSON schemas for latest models starting from GPT-4o.
+                  Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                  For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+                - For structured outputs with streaming,
+                  the `response_format` must be a JSON schema and not a Pydantic model.
+        :param default_headers: Default headers to use for the AzureOpenAI client.
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param azure_ad_token_provider: A function that returns an Azure Active Directory token, will be invoked on
+            every request.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        # We intentionally do not call super().__init__ here because we only need to instantiate the client to interact
+        # with the API.
+
+        # Why is this here?
+        # AzureOpenAI init is forcing us to use an init method that takes either base_url or azure_endpoint as not
+        # None init parameters. This way we accommodate the use case where env var AZURE_OPENAI_ENDPOINT is set instead
+        # of passing it as a parameter.
+        azure_endpoint = azure_endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")
+        # `azure_endpoint` accepts either a plain string or a `Secret`. We keep the original value on the instance for
+        # serialization and resolve it to a string only to validate that an endpoint was provided.
+        resolved_azure_endpoint = (
+            azure_endpoint.resolve_value() if isinstance(azure_endpoint, Secret) else azure_endpoint
+        )
+        if not resolved_azure_endpoint:
+            raise ValueError("Please provide an Azure endpoint or set the environment variable AZURE_OPENAI_ENDPOINT.")
+
+        if api_key is None and azure_ad_token is None:
+            raise ValueError("Please provide an API key or an Azure Active Directory token.")
+
+        # The check above makes mypy incorrectly infer that api_key is never None,
+        # which propagates the incorrect type.
+        self.api_key = api_key  # type: ignore
+        self.azure_ad_token = azure_ad_token
+        self.generation_kwargs = generation_kwargs or {}
+        self.streaming_callback = streaming_callback
+        self.api_version = api_version
+        self.azure_endpoint = azure_endpoint
+        self.azure_deployment = azure_deployment
+        self.organization = organization
+        self.model = azure_deployment or "gpt-4.1-mini"
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.default_headers = default_headers or {}
+        self.azure_ad_token_provider = azure_ad_token_provider
+        self.http_client_kwargs = http_client_kwargs
+        _check_duplicate_tool_names(flatten_tools_or_toolsets(tools))
+        self.tools = tools
+        self.tools_strict = tools_strict
+
+        self.client: AzureOpenAI | None = None
+        self.async_client: AsyncAzureOpenAI | None = None
+        self._tools_warmed_up = False
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        resolved_azure_endpoint = (
+            self.azure_endpoint.resolve_value() if isinstance(self.azure_endpoint, Secret) else self.azure_endpoint
+        )
+        resolved_api_version = (
+            self.api_version.resolve_value() if isinstance(self.api_version, Secret) else self.api_version
+        )
+        return {
+            "api_version": resolved_api_version,
+            "azure_endpoint": resolved_azure_endpoint,
+            "azure_deployment": self.azure_deployment,
+            "api_key": self.api_key.resolve_value() if self.api_key is not None else None,
+            "azure_ad_token": self.azure_ad_token.resolve_value() if self.azure_ad_token is not None else None,
+            "organization": self.organization,
+            "timeout": timeout,
+            "max_retries": max_retries,
+            "default_headers": self.default_headers,
+            "azure_ad_token_provider": self.azure_ad_token_provider,
+        }
+
+    def _warm_up_tools(self) -> None:
+        if not self._tools_warmed_up:
+            warm_up_tools(self.tools)
+            self._tools_warmed_up = True
+
+    def warm_up(self) -> None:
+        """
+        Warm up the tools and initialize the synchronous Azure OpenAI client.
+        """
+        self._warm_up_tools()
+        if self.client is None:
+            self.client = AzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Warm up the tools and initialize the asynchronous Azure OpenAI client on the serving event loop.
+        """
+        self._warm_up_tools()
+        if self.async_client is None:
+            self.async_client = AsyncAzureOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous Azure OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous Azure OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        azure_ad_token_provider_name = None
+        if self.azure_ad_token_provider:
+            azure_ad_token_provider_name = serialize_callable(self.azure_ad_token_provider)
+        # If the response format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        generation_kwargs = self.generation_kwargs.copy()
+        response_format = generation_kwargs.get("response_format")
+        if response_format and issubclass(response_format, BaseModel):
+            json_schema = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": response_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(response_format),
+                },
+            }
+            generation_kwargs["response_format"] = json_schema
+        return default_to_dict(
+            self,
+            azure_endpoint=self.azure_endpoint.to_dict()
+            if isinstance(self.azure_endpoint, Secret)
+            else self.azure_endpoint,
+            azure_deployment=self.azure_deployment,
+            organization=self.organization,
+            api_version=self.api_version.to_dict() if isinstance(self.api_version, Secret) else self.api_version,
+            streaming_callback=callback_name,
+            generation_kwargs=generation_kwargs,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            api_key=self.api_key,
+            azure_ad_token=self.azure_ad_token,
+            default_headers=self.default_headers,
+            tools=serialize_tools_or_toolset(self.tools),
+            tools_strict=self.tools_strict,
+            azure_ad_token_provider=azure_ad_token_provider_name,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        serialized_azure_ad_token_provider = init_params.get("azure_ad_token_provider")
+        if serialized_azure_ad_token_provider:
+            data["init_parameters"]["azure_ad_token_provider"] = deserialize_callable(
+                serialized_azure_ad_token_provider
+            )
+        return default_from_dict(cls, data)

haystack/components/generators/chat/azure_responses.py

@@ -0,0 +1,272 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from collections.abc import Awaitable, Callable
+from typing import Any, ClassVar
+
+from openai.lib._pydantic import to_strict_json_schema
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.components.generators.chat import OpenAIResponsesChatGenerator
+from haystack.dataclasses.streaming_chunk import StreamingCallbackT
+from haystack.tools import ToolsType, deserialize_tools_or_toolset_inplace, serialize_tools_or_toolset
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+
+
+@component
+class AzureOpenAIResponsesChatGenerator(OpenAIResponsesChatGenerator):
+    """
+    Completes chats using OpenAI's Responses API on Azure.
+
+    It works with the gpt-5 and o-series models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.Responses.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/responses).
+
+    ### Usage example
+    <!-- test-ignore -->
+    ```python
+    from haystack.components.generators.chat import AzureOpenAIResponsesChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = AzureOpenAIResponsesChatGenerator(
+        azure_endpoint="https://example-resource.azure.openai.com/",
+        generation_kwargs={"reasoning": {"effort": "low", "summary": "auto"}}
+    )
+    response = client.run(messages)
+    print(response)
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5.4-pro",
+        "gpt-5.4",
+        "gpt-5.3-chat",
+        "gpt-5.3-codex",
+        "gpt-5.2-codex",
+        "gpt-5.2",
+        "gpt-5.2-chat",
+        "gpt-5.1-codex-max",
+        "gpt-5.1",
+        "gpt-5.1-chat",
+        "gpt-5.1-codex",
+        "gpt-5.1-codex-mini",
+        "gpt-5-pro",
+        "gpt-5-codex",
+        "gpt-5",
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5-chat",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "computer-use-preview",
+        "gpt-4.1",
+        "gpt-4.1-nano",
+        "gpt-4.1-mini",
+        "gpt-image-1",
+        "gpt-image-1-mini",
+        "gpt-image-1.5",
+        "o1",
+        "o3-mini",
+        "o3",
+        "o4-mini",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses#model-support for the full list."""
+
+    # ruff: noqa: PLR0913
+    def __init__(
+        self,
+        *,
+        api_key: Secret | Callable[[], str] | Callable[[], Awaitable[str]] = Secret.from_env_var(
+            "AZURE_OPENAI_API_KEY", strict=False
+        ),
+        azure_endpoint: str | None = None,
+        azure_deployment: str = "gpt-5-mini",
+        streaming_callback: StreamingCallbackT | None = None,
+        organization: str | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initialize the AzureOpenAIResponsesChatGenerator component.
+
+        :param api_key: The API key to use for authentication. Can be:
+            - A `Secret` object containing the API key.
+            - A `Secret` object containing the [Azure Active Directory token](https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id).
+            - A function that returns an Azure Active Directory token.
+        :param azure_endpoint: The endpoint of the deployed model, for example `"https://example-resource.azure.openai.com/"`.
+        :param azure_deployment: The deployment of the model, usually the model name.
+        :param organization: Your organization ID, defaults to `None`. For help, see
+        [Setting up your organization](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param streaming_callback: A callback function called when a new token is received from the stream.
+            It accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param timeout: Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries: Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent
+           directly to the OpenAI endpoint.
+           See OpenAI [documentation](https://platform.openai.com/docs/api-reference/responses) for
+            more details.
+            Some of the supported parameters:
+            - `temperature`: What sampling temperature to use. Higher values like 0.8 will make the output more random,
+                while lower values like 0.2 will make it more focused and deterministic.
+            - `top_p`: An alternative to sampling with temperature, called nucleus sampling, where the model
+                considers the results of the tokens with top_p probability mass. For example, 0.1 means only the tokens
+                comprising the top 10% probability mass are considered.
+            - `previous_response_id`: The ID of the previous response.
+                Use this to create multi-turn conversations.
+            - `text_format`: A Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+            - `text`: A JSON schema that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                Notes:
+                - Both JSON Schema and Pydantic models are supported for latest models starting from GPT-4o.
+                - If both are provided, `text_format` takes precedence and json schema passed to `text` is ignored.
+                - Currently, this component doesn't support streaming for structured outputs.
+                - Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                    For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+            - `reasoning`: A dictionary of parameters for reasoning. For example:
+                - `summary`: The summary of the reasoning.
+                - `effort`: The level of effort to put into the reasoning. Can be `low`, `medium` or `high`.
+                - `generate_summary`: Whether to generate a summary of the reasoning.
+                Note: OpenAI does not return the reasoning tokens, but we can view summary if its enabled.
+                For details, see the [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning).
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        azure_endpoint = azure_endpoint or os.getenv("AZURE_OPENAI_ENDPOINT")
+        if azure_endpoint is None:
+            raise ValueError(
+                "You must provide `azure_endpoint` or set the `AZURE_OPENAI_ENDPOINT` environment variable."
+            )
+        self._azure_endpoint = azure_endpoint
+        self._azure_deployment = azure_deployment
+        super(AzureOpenAIResponsesChatGenerator, self).__init__(  # noqa: UP008
+            api_key=api_key,  # type: ignore[arg-type]
+            model=self._azure_deployment,
+            streaming_callback=streaming_callback,
+            api_base_url=f"{self._azure_endpoint.rstrip('/')}/openai/v1",
+            organization=organization,
+            generation_kwargs=generation_kwargs,
+            timeout=timeout,
+            max_retries=max_retries,
+            tools=tools,
+            tools_strict=tools_strict,
+            http_client_kwargs=http_client_kwargs,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+
+        # API key can be a secret or a callable
+        serialized_api_key = (
+            serialize_callable(self.api_key)
+            if callable(self.api_key)
+            else self.api_key.to_dict()
+            if isinstance(self.api_key, Secret)
+            else None
+        )
+
+        # If the text format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        generation_kwargs = self.generation_kwargs.copy()
+        text_format = generation_kwargs.pop("text_format", None)
+        if text_format and isinstance(text_format, type) and issubclass(text_format, BaseModel):
+            json_schema = {
+                "format": {
+                    "type": "json_schema",
+                    "name": text_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(text_format),
+                }
+            }
+            # json schema needs to be passed to text parameter instead of text_format
+            generation_kwargs["text"] = json_schema
+
+        # OpenAI/MCP tools are passed as list of dictionaries
+        serialized_tools: dict[str, Any] | list[dict[str, Any]] | None
+        if self.tools and isinstance(self.tools, list) and isinstance(self.tools[0], dict):
+            # mypy can't infer that self.tools is list[dict] here
+            serialized_tools = self.tools
+        else:
+            serialized_tools = serialize_tools_or_toolset(self.tools)  # type: ignore[arg-type]
+
+        return default_to_dict(
+            self,
+            azure_endpoint=self._azure_endpoint,
+            api_key=serialized_api_key,
+            azure_deployment=self._azure_deployment,
+            streaming_callback=callback_name,
+            organization=self.organization,
+            generation_kwargs=generation_kwargs,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            tools=serialized_tools,
+            tools_strict=self.tools_strict,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIResponsesChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        # If api_key is a str, it's a callable (Secrets are handled automatically by default_from_dict)
+        serialized_api_key = data["init_parameters"].get("api_key")
+        if isinstance(serialized_api_key, str):
+            data["init_parameters"]["api_key"] = deserialize_callable(serialized_api_key)
+
+        # we only deserialize the tools if they are haystack tools
+        # because openai tools are not serialized in the same way
+        tools = data["init_parameters"].get("tools")
+        if tools and (
+            isinstance(tools, dict)
+            and tools.get("type") == "haystack.tools.toolset.Toolset"
+            or isinstance(tools, list)
+            and tools[0].get("type") == "haystack.tools.tool.Tool"
+        ):
+            deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        return default_from_dict(cls, data)

haystack/components/generators/chat/fallback.py

@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.components.generators.utils import _normalize_messages
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ChatMessage, StreamingCallbackT
+from haystack.tools import ToolsType
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.deserialization import deserialize_component_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class FallbackChatGenerator:
+    """
+    A chat generator wrapper that tries multiple chat generators sequentially.
+
+    It forwards all parameters transparently to the underlying chat generators and returns the first successful result.
+    Calls chat generators sequentially until one succeeds. Falls back on any exception raised by a generator.
+    If all chat generators fail, it raises a RuntimeError with details.
+
+    Timeout enforcement is fully delegated to the underlying chat generators. The fallback mechanism will only
+    work correctly if the underlying chat generators implement proper timeout handling and raise exceptions
+    when timeouts occur. For predictable latency guarantees, ensure your chat generators:
+    - Support a `timeout` parameter in their initialization
+    - Implement timeout as total wall-clock time (shared deadline for both streaming and non-streaming)
+    - Raise timeout exceptions (e.g., TimeoutError, asyncio.TimeoutError, httpx.TimeoutException) when exceeded
+
+    Note: Most well-implemented chat generators (OpenAI, Anthropic, Cohere, etc.) support timeout parameters
+    with consistent semantics. For HTTP-based LLM providers, a single timeout value (e.g., `timeout=30`)
+    typically applies to all connection phases: connection setup, read, write, and pool. For streaming
+    responses, read timeout is the maximum gap between chunks. For non-streaming, it's the time limit for
+    receiving the complete response.
+
+    Fail over is automatically triggered when a generator raises any exception, including:
+    - Timeout errors (if the generator implements and raises them)
+    - Rate limit errors (429)
+    - Authentication errors (401)
+    - Context length errors (400)
+    - Server errors (500+)
+    - Any other exception
+    """
+
+    def __init__(self, chat_generators: list[ChatGenerator]) -> None:
+        """
+        Creates an instance of FallbackChatGenerator.
+
+        :param chat_generators: A non-empty list of chat generator components to try in order.
+        """
+        if not chat_generators:
+            msg = "'chat_generators' must be a non-empty list"
+            raise ValueError(msg)
+
+        self.chat_generators = list(chat_generators)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component, including nested chat generators."""
+        return default_to_dict(
+            self,
+            chat_generators=[
+                component_to_dict(gen, name=f"chat_generator_{idx}") for idx, gen in enumerate(self.chat_generators)
+            ],
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FallbackChatGenerator:
+        """Rebuild the component from a serialized representation, restoring nested chat generators."""
+        # Reconstruct nested chat generators from their serialized dicts
+        init_params = data.get("init_parameters", {})
+        serialized = init_params.get("chat_generators") or []
+        deserialized: list[Any] = []
+        for g in serialized:
+            # Use the generic component deserializer available in Haystack
+            holder = {"component": g}
+            deserialize_component_inplace(holder, key="component")
+            deserialized.append(holder["component"])
+        init_params["chat_generators"] = deserialized
+        data["init_parameters"] = init_params
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """Warm up all underlying chat generators."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "warm_up"):
+                gen.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """Warm up all underlying chat generators on the serving event loop."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "warm_up_async"):
+                await gen.warm_up_async()
+            elif hasattr(gen, "warm_up"):
+                gen.warm_up()
+
+    def close(self) -> None:
+        """Release the underlying chat generators' resources."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "close"):
+                gen.close()
+
+    async def close_async(self) -> None:
+        """Release the underlying chat generators' async resources."""
+        for gen in self.chat_generators:
+            if hasattr(gen, "close_async"):
+                await gen.close_async()
+            elif hasattr(gen, "close"):
+                gen.close()
+
+    def _run_single_sync(
+        self,
+        gen: Any,
+        messages: list[ChatMessage],
+        generation_kwargs: dict[str, Any] | None,
+        tools: ToolsType | None,
+        streaming_callback: StreamingCallbackT | None,
+    ) -> dict[str, Any]:
+        return gen.run(
+            messages=messages, generation_kwargs=generation_kwargs, tools=tools, streaming_callback=streaming_callback
+        )
+
+    async def _run_single_async(
+        self,
+        gen: Any,
+        messages: list[ChatMessage],
+        generation_kwargs: dict[str, Any] | None,
+        tools: ToolsType | None,
+        streaming_callback: StreamingCallbackT | None,
+    ) -> dict[str, Any]:
+        return await _execute_component_async(
+            gen,
+            messages=messages,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            streaming_callback=streaming_callback,
+        )
+
+    @component.output_types(replies=list[ChatMessage], meta=dict[str, Any])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> dict[str, list[ChatMessage] | dict[str, Any]]:
+        """
+        Execute chat generators sequentially until one succeeds.
+
+        :param messages: The conversation history as a list of ChatMessage instances.
+        :param generation_kwargs: Optional parameters for the chat generator (e.g., temperature, max_tokens).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for function calling capabilities.
+        :param streaming_callback: Optional callable for handling streaming responses.
+        :returns: A dictionary with:
+            - "replies": Generated ChatMessage instances from the first successful generator.
+            - "meta": Execution metadata including successful_chat_generator_index, successful_chat_generator_class,
+              total_attempts, failed_chat_generators, plus any metadata from the successful generator.
+        :raises RuntimeError: If all chat generators fail.
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+
+        failed: list[str] = []
+        last_error: BaseException | None = None
+
+        for idx, gen in enumerate(self.chat_generators):
+            gen_name = gen.__class__.__name__
+            try:
+                result = self._run_single_sync(gen, messages, generation_kwargs, tools, streaming_callback)
+                replies = result.get("replies", [])
+                meta = dict(result.get("meta", {}))
+                meta.update(
+                    {
+                        "successful_chat_generator_index": idx,
+                        "successful_chat_generator_class": gen_name,
+                        "total_attempts": idx + 1,
+                        "failed_chat_generators": failed,
+                    }
+                )
+                return {"replies": replies, "meta": meta}
+            except Exception as e:  # noqa: BLE001 - fallback logic should handle any exception
+                logger.warning(
+                    "ChatGenerator {chat_generator} failed with error: {error}", chat_generator=gen_name, error=e
+                )
+                failed.append(gen_name)
+                last_error = e
+
+        failed_names = ", ".join(failed)
+        msg = (
+            f"All {len(self.chat_generators)} chat generators failed. "
+            f"Last error: {last_error}. Failed chat generators: [{failed_names}]"
+        )
+        raise RuntimeError(msg)
+
+    @component.output_types(replies=list[ChatMessage], meta=dict[str, Any])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> dict[str, list[ChatMessage] | dict[str, Any]]:
+        """
+        Asynchronously execute chat generators sequentially until one succeeds.
+
+        :param messages: The conversation history as a list of ChatMessage instances.
+        :param generation_kwargs: Optional parameters for the chat generator (e.g., temperature, max_tokens).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for function calling capabilities.
+        :param streaming_callback: Optional callable for handling streaming responses.
+        :returns: A dictionary with:
+            - "replies": Generated ChatMessage instances from the first successful generator.
+            - "meta": Execution metadata including successful_chat_generator_index, successful_chat_generator_class,
+              total_attempts, failed_chat_generators, plus any metadata from the successful generator.
+        :raises RuntimeError: If all chat generators fail.
+        """
+        await self.warm_up_async()
+
+        messages = _normalize_messages(messages)
+
+        failed: list[str] = []
+        last_error: BaseException | None = None
+
+        for idx, gen in enumerate(self.chat_generators):
+            gen_name = gen.__class__.__name__
+            try:
+                result = await self._run_single_async(gen, messages, generation_kwargs, tools, streaming_callback)
+                replies = result.get("replies", [])
+                meta = dict(result.get("meta", {}))
+                meta.update(
+                    {
+                        "successful_chat_generator_index": idx,
+                        "successful_chat_generator_class": gen_name,
+                        "total_attempts": idx + 1,
+                        "failed_chat_generators": failed,
+                    }
+                )
+                return {"replies": replies, "meta": meta}
+            except Exception as e:  # noqa: BLE001 - fallback logic should handle any exception
+                logger.warning(
+                    "ChatGenerator {chat_generator} failed with error: {error}", chat_generator=gen_name, error=e
+                )
+                failed.append(gen_name)
+                last_error = e
+
+        failed_names = ", ".join(failed)
+        msg = (
+            f"All {len(self.chat_generators)} chat generators failed. "
+            f"Last error: {last_error}. Failed chat generators: [{failed_names}]"
+        )
+        raise RuntimeError(msg)

haystack/components/generators/chat/llm.py

@@ -0,0 +1,205 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Literal
+
+from haystack import component, logging
+from haystack.components.agents.agent import Agent
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage, StreamingCallbackT
+from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
+from haystack.utils.deserialization import deserialize_component_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLM(Agent):
+    """
+    A text generation component powered by a large language model.
+
+    The LLM component is a simplified version of the Agent that focuses solely on text generation
+    without tool usage. It processes messages and returns a single response from the language model.
+
+    ### Usage examples
+    ```python
+    from haystack.components.generators.chat import LLM
+    from haystack.components.generators.chat import OpenAIChatGenerator
+
+    llm = LLM(
+        chat_generator=OpenAIChatGenerator(),
+        system_prompt="You are a helpful translation assistant.",
+        user_prompt="Summarize the following document: {{ document }}",
+        required_variables=["document"],
+    )
+
+    result = llm.run(document="The weather is lovely today and the sun is shining. ")
+    print(result["last_message"].text)
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator,
+        system_prompt: str | None = None,
+        user_prompt: str | None = None,
+        required_variables: list[str] | Literal["*"] = "*",
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> None:
+        """
+        Initialize the LLM component.
+
+        :param chat_generator: An instance of the chat generator that the LLM should use.
+        :param system_prompt: System prompt for the LLM. Can be a plain string template or a Jinja2 message template.
+        :param user_prompt: User prompt for the LLM. This prompt is appended to the messages provided at
+            runtime. Can be a plain string template or a Jinja2 message template. If it contains template variables
+            (e.g., `{{ variable_name }}`), they become inputs to the component. If omitted or if there are no
+            template variables, `messages` must be provided at runtime instead.
+        :param required_variables:
+            Variables that must be provided as input to `user_prompt` or `system_prompt`.
+            If a variable listed as required is not provided, an exception is raised.
+            If set to `"*"`, all variables found in the prompt are required. Defaults to `"*"`.
+            Only relevant when `user_prompt` or `system_prompt` contains template variables.
+        :param streaming_callback: A callback that will be invoked when a response is streamed from the LLM.
+        :raises ValueError: If user_prompt contains template variables but required_variables is an empty list.
+        """
+        super(LLM, self).__init__(  # noqa: UP008
+            chat_generator=chat_generator,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            required_variables=required_variables,
+            streaming_callback=streaming_callback,
+        )
+        if self._user_chat_prompt_builder is None or len(self._user_chat_prompt_builder.variables) == 0:
+            # This means user_prompt is empty or has no template variables.
+            # To ensure properly scheduling we then require messages to be passed at runtime.
+            component.set_input_type(self, "messages", list[ChatMessage])
+        else:
+            # user prompt was provided with variables
+            if isinstance(required_variables, list) and len(required_variables) == 0:
+                raise ValueError(
+                    "required_variables must not be empty. Set it to '*' to require all variables, "
+                    "or provide a non-empty list of variable names."
+                )
+            component.set_input_type(self, "messages", list[ChatMessage], None)
+
+        # The Agent base class declares `step_count` and `tool_call_counts` as outputs, but an LLM never has tools
+        # and always runs exactly one step — those values are uninformative, so drop them from the public surface.
+        # `token_usage` is still meaningful and stays exposed.
+        component.set_output_types(
+            self, messages=list[ChatMessage], last_message=ChatMessage, token_usage=dict[str, Any]
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize the LLM component to a dictionary.
+
+        :return: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self.chat_generator, name="chat_generator"),
+            system_prompt=self.system_prompt,
+            user_prompt=self.user_prompt,
+            required_variables=self.required_variables,
+            streaming_callback=serialize_callable(self.streaming_callback) if self.streaming_callback else None,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLM":
+        """
+        Deserialize the LLM from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :return: Deserialized LLM instance.
+        """
+        init_params = data.get("init_parameters", {})
+
+        deserialize_component_inplace(init_params, key="chat_generator")
+
+        if init_params.get("streaming_callback") is not None:
+            init_params["streaming_callback"] = deserialize_callable(init_params["streaming_callback"])
+
+        return default_from_dict(cls, data)
+
+    def run(  # type: ignore[override]  # `messages` is in **kwargs to allow dynamic required/optional status
+        self,
+        *,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Process messages and generate a response from the language model.
+
+        :param messages: Optional list of ChatMessage objects to prepend to the conversation. Whether this is
+            required or optional depends on the `user_prompt` configuration: if `user_prompt` has no template
+            variables, `messages` must be provided. Passed via `**kwargs`.
+        :param streaming_callback: A callback that will be invoked when a response is streamed from the LLM.
+        :param generation_kwargs: Additional keyword arguments for the underlying chat generator. These parameters
+            will override the parameters passed during component initialization.
+        :param kwargs: Additional keyword arguments. These are used to fill template variables in `user_prompt` or
+            `system_prompt` (the keys must match template variable names).
+        :returns:
+            A dictionary with the following keys:
+            - "messages": List of all messages exchanged during the LLM's run.
+            - "last_message": The last message exchanged during the LLM's run.
+            - "token_usage": Token usage from the LLM call (e.g. prompt_tokens, completion_tokens). Empty if the
+              chat generator did not return usage data.
+        """
+        # `messages` is intentionally omitted from the signature so the framework can treat it as required
+        # or optional depending on init configuration. See __init__ for details.
+        messages = kwargs.pop("messages", None)
+        result = super(LLM, self).run(  # noqa: UP008
+            messages=messages or [],
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            **kwargs,
+        )
+        # Inherited Agent-internal bookkeeping that isn't useful at the LLM surface.
+        result.pop("step_count", None)
+        result.pop("tool_call_counts", None)
+        return result
+
+    async def run_async(  # type: ignore[override]  # `messages` is in **kwargs to allow dynamic required/optional status
+        self,
+        *,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Asynchronously process messages and generate a response from the language model.
+
+        :param messages: Optional list of ChatMessage objects to prepend to the conversation. Whether this is
+            required or optional depends on the `user_prompt` configuration: if `user_prompt` has no template
+            variables, `messages` must be provided. Passed via `**kwargs`.
+        :param streaming_callback: An asynchronous callback that will be invoked when a response is streamed
+            from the LLM.
+        :param generation_kwargs: Additional keyword arguments for the underlying chat generator. These parameters
+            will override the parameters passed during component initialization.
+        :param kwargs: Additional keyword arguments. These are used to fill template variables in `user_prompt` or
+            `system_prompt` (the keys must match template variable names).
+        :returns:
+            A dictionary with the following keys:
+            - "messages": List of all messages exchanged during the LLM's run.
+            - "last_message": The last message exchanged during the LLM's run.
+            - "token_usage": Token usage from the LLM call (e.g. prompt_tokens, completion_tokens). Empty if the
+              chat generator did not return usage data.
+        """
+        # `messages` is intentionally omitted from the signature so the framework can treat it as required
+        # or optional depending on init configuration. See __init__ for details.
+        messages = kwargs.pop("messages", None)
+        result = await super(LLM, self).run_async(  # noqa: UP008
+            messages=messages or [],
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            **kwargs,
+        )
+        # Inherited Agent-internal bookkeeping that isn't useful at the LLM surface.
+        result.pop("step_count", None)
+        result.pop("tool_call_counts", None)
+        return result

haystack/components/generators/chat/mock.py

@@ -0,0 +1,374 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable, Sequence
+from dataclasses import replace
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.utils import _normalize_messages
+from haystack.dataclasses import (
+    ChatMessage,
+    ChatRole,
+    ComponentInfo,
+    FinishReason,
+    StreamingCallbackT,
+    StreamingChunk,
+    select_streaming_callback,
+)
+from haystack.dataclasses.streaming_chunk import ToolCallDelta, _invoke_streaming_callback
+from haystack.tools import ToolsType
+from haystack.utils import deserialize_callable, serialize_callable
+
+logger = logging.getLogger(__name__)
+
+# A callable that derives a response from the input messages. It receives the (normalized) list of input
+# `ChatMessage` objects and returns either the text of the assistant reply or a full `ChatMessage`.
+ResponseFn = Callable[[list[ChatMessage]], str | ChatMessage]
+
+
+@component
+class MockChatGenerator:
+    """
+    A Chat Generator that returns predefined responses without calling any API.
+
+    It is a drop-in replacement for real Chat Generators (such as `OpenAIChatGenerator`) in tests, smoke tests, and
+    quick prototypes. It implements the same interface (`run`, `run_async`, streaming, serialization) but never
+    contacts an external service, so it is fully deterministic and free to run.
+
+    The response is selected based on how the component is configured:
+
+    - **Fixed response**: pass a single string or `ChatMessage`. The same reply is returned on every call.
+      Any `ChatMessage` passed as a response must have the `assistant` role.
+    - **Cycling responses**: pass a list of strings and/or `ChatMessage` objects. Each call returns the next item,
+      wrapping around to the start once the list is exhausted. This is useful to drive multi-step flows such as
+      Agents, where the first call returns a tool call and a later call returns the final answer.
+    - **Dynamic response**: pass a `response_fn` callable that receives the input messages and returns the reply.
+      This is useful when the reply should depend on the input, for example to echo back part of the prompt.
+    - **Echo (default)**: with no configuration, the component echoes back the text of the last message that has
+      text content. This makes it usable out of the box for quick prototyping.
+
+    Pass `ChatMessage` objects (rather than plain strings) to return tool calls or reasoning content, which is handy
+    for exercising tool-calling pipelines without a real model.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.generators.chat import MockChatGenerator
+    from haystack.dataclasses import ChatMessage, ToolCall
+
+    # Fixed response
+    generator = MockChatGenerator(responses="Hello, this is a mock response.")
+    result = generator.run([ChatMessage.from_user("Hi!")])
+    print(result["replies"][0].text)  # "Hello, this is a mock response."
+
+    # Cycling responses to drive an Agent-like loop
+    generator = MockChatGenerator(
+        responses=[
+            ChatMessage.from_assistant(tool_calls=[ToolCall(tool_name="search", arguments={"query": "Haystack"})]),
+            "Here is the final answer.",
+        ]
+    )
+    ```
+    """
+
+    def __init__(
+        self,
+        responses: str | ChatMessage | Sequence[str | ChatMessage] | None = None,
+        *,
+        response_fn: ResponseFn | None = None,
+        model: str = "mock-model",
+        meta: dict[str, Any] | None = None,
+        streaming_callback: StreamingCallbackT | None = None,
+    ) -> None:
+        """
+        Creates an instance of MockChatGenerator.
+
+        :param responses: The predefined response(s) to return. Accepts a single string or `ChatMessage` (returned on
+            every call), or a non-empty list of strings and/or `ChatMessage` objects that are returned in order,
+            cycling back to the start once exhausted. Strings are wrapped into assistant `ChatMessage` objects, and any
+            `ChatMessage` passed must have the `assistant` role. Mutually exclusive with `response_fn`. If neither is
+            provided, the component echoes the last message with text content.
+        :param response_fn: An optional callable that receives the input messages and returns the reply as a string or
+            an assistant `ChatMessage`. Use this for input-dependent responses. Mutually exclusive with `responses`. To
+            support serialization, pass a named function (lambdas and nested functions cannot be serialized).
+        :param model: The model name reported in the response metadata. Purely cosmetic; no model is loaded.
+        :param meta: Additional metadata merged into the `meta` of every returned `ChatMessage`. A per-response
+            `ChatMessage`'s own metadata takes precedence over this value.
+        :param streaming_callback: An optional callback invoked with `StreamingChunk` objects reconstructed from the
+            predefined response. It lets the mock exercise streaming code paths without a real model.
+        :raises ValueError: If both `responses` and `response_fn` are provided, if `responses` is an empty list, or if
+            a `ChatMessage` response does not have the `assistant` role.
+        """
+        if responses is not None and response_fn is not None:
+            raise ValueError("Pass either 'responses' or 'response_fn', not both.")
+
+        self._responses = self._normalize_responses(responses)
+        self.response_fn = response_fn
+        self.model = model
+        self.meta = meta or {}
+        self.streaming_callback = streaming_callback
+        self._call_count = 0
+        self._is_warmed_up = False
+
+    @staticmethod
+    def _normalize_responses(
+        responses: str | ChatMessage | Sequence[str | ChatMessage] | None,
+    ) -> list[ChatMessage] | None:
+        """Normalize the `responses` argument into a non-empty list of `ChatMessage`, or `None` for echo mode."""
+        if responses is None:
+            return None
+
+        items: list[str | ChatMessage]
+        if isinstance(responses, (str, ChatMessage)):
+            items = [responses]
+        elif isinstance(responses, Sequence):
+            items = list(responses)
+        else:
+            raise TypeError(f"'responses' must be a string, ChatMessage, or a sequence of them, got {type(responses)}.")
+
+        if len(items) == 0:
+            raise ValueError("'responses' must not be an empty list.")
+
+        normalized: list[ChatMessage] = []
+        for item in items:
+            if isinstance(item, str):
+                normalized.append(ChatMessage.from_assistant(item))
+            elif isinstance(item, ChatMessage):
+                if item.role != ChatRole.ASSISTANT:
+                    raise ValueError(
+                        f"Each ChatMessage response must have the 'assistant' role, got '{item.role.value}'."
+                    )
+                normalized.append(item)
+            else:
+                raise TypeError(f"Each response must be a string or ChatMessage, got {type(item)}.")
+        return normalized
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the component to a dictionary."""
+        responses = [msg.to_dict() for msg in self._responses] if self._responses is not None else None
+        response_fn = serialize_callable(self.response_fn) if self.response_fn is not None else None
+        streaming_callback = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        return default_to_dict(
+            self,
+            responses=responses,
+            response_fn=response_fn,
+            model=self.model,
+            meta=self.meta,
+            streaming_callback=streaming_callback,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> MockChatGenerator:
+        """Deserialize the component from a dictionary."""
+        init_params = data.get("init_parameters", {})
+        responses = init_params.get("responses")
+        if responses is not None:
+            init_params["responses"] = [ChatMessage.from_dict(msg) for msg in responses]
+        response_fn = init_params.get("response_fn")
+        if response_fn:
+            init_params["response_fn"] = deserialize_callable(response_fn)
+        streaming_callback = init_params.get("streaming_callback")
+        if streaming_callback:
+            init_params["streaming_callback"] = deserialize_callable(streaming_callback)
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """No-op warm up, provided for interface compatibility with real Chat Generators."""
+        self._is_warmed_up = True
+
+    @staticmethod
+    def _echo_text(messages: list[ChatMessage]) -> str | None:
+        """Return the text of the last message that has text content, for echo mode."""
+        for message in reversed(messages):
+            if message.text:
+                return message.text
+        return None
+
+    @staticmethod
+    def _coerce_to_message(result: str | ChatMessage) -> ChatMessage:
+        """Turn the output of `response_fn` into a `ChatMessage`, wrapping strings and requiring the assistant role."""
+        if isinstance(result, str):
+            return ChatMessage.from_assistant(result)
+        if isinstance(result, ChatMessage):
+            if result.role != ChatRole.ASSISTANT:
+                raise ValueError(f"'response_fn' must return an assistant ChatMessage, got '{result.role.value}'.")
+            return result
+        raise TypeError(f"'response_fn' must return a string or ChatMessage, got {type(result)}.")
+
+    @staticmethod
+    def _estimate_usage(messages: list[ChatMessage], reply: ChatMessage) -> dict[str, int]:
+        """
+        Roughly estimate token usage as whitespace-separated word counts.
+
+        This is an approximation (not real tokenization) intended to give downstream code realistic-looking metadata.
+        """
+        prompt_tokens = sum(len((message.text or "").split()) for message in messages)
+        completion_tokens = len((reply.text or "").split())
+        return {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        }
+
+    def _build_meta(self, messages: list[ChatMessage], base: ChatMessage) -> dict[str, Any]:
+        """Build the metadata attached to the returned reply, merging defaults, init meta, and per-response meta."""
+        meta: dict[str, Any] = {
+            "model": self.model,
+            "index": 0,
+            "finish_reason": "tool_calls" if base.tool_calls else "stop",
+            "usage": self._estimate_usage(messages, base),
+        }
+        meta.update(self.meta)
+        meta.update(base.meta)
+        return meta
+
+    def _build_reply(self, messages: list[ChatMessage]) -> ChatMessage | None:
+        """Select and finalize the reply for the given input messages. Returns `None` when there is nothing to echo."""
+        if self.response_fn is not None:
+            base = self._coerce_to_message(self.response_fn(messages))
+        elif self._responses is not None:
+            base = self._responses[self._call_count % len(self._responses)]
+            self._call_count += 1
+        else:
+            text = self._echo_text(messages)
+            if text is None:
+                return None
+            base = ChatMessage.from_assistant(text)
+
+        return replace(base, _meta=self._build_meta(messages, base))
+
+    def _make_chunks(self, reply: ChatMessage) -> list[StreamingChunk]:
+        """Reconstruct streaming chunks from a finalized reply so streaming callbacks can be exercised."""
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+
+        # Stream the text content word by word in content block 0.
+        parts = re.findall(r"\S+\s*", reply.text) if reply.text else []
+        for idx, part in enumerate(parts):
+            chunks.append(
+                StreamingChunk(
+                    content=part, component_info=component_info, index=0, start=(idx == 0), meta={"model": self.model}
+                )
+            )
+
+        # Stream each tool call in its own content block.
+        block_index = 1 if parts else 0
+        for tool_call in reply.tool_calls:
+            chunks.append(
+                StreamingChunk(
+                    content="",
+                    component_info=component_info,
+                    index=block_index,
+                    start=True,
+                    tool_calls=[
+                        ToolCallDelta(
+                            index=block_index,
+                            tool_name=tool_call.tool_name,
+                            arguments=json.dumps(tool_call.arguments),
+                            id=tool_call.id,
+                        )
+                    ],
+                    meta={"model": self.model},
+                )
+            )
+            block_index += 1
+
+        if not chunks:
+            chunks.append(
+                StreamingChunk(content="", component_info=component_info, index=0, meta={"model": self.model})
+            )
+
+        finish_reason: FinishReason = "tool_calls" if reply.tool_calls else "stop"
+        last = chunks[-1]
+        chunks[-1] = replace(last, finish_reason=finish_reason, meta={**last.meta, "finish_reason": finish_reason})
+        return chunks
+
+    @component.output_types(replies=list[ChatMessage])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,  # noqa: ARG002
+        *,
+        tools: ToolsType | None = None,  # noqa: ARG002
+        tools_strict: bool | None = None,  # noqa: ARG002
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Return a predefined reply for the given messages without calling any API.
+
+        The signature mirrors `OpenAIChatGenerator.run` so the mock can be used as a positional drop-in replacement.
+
+        :param messages: The conversation history as a list of `ChatMessage` instances or a single string.
+        :param streaming_callback: An optional callback invoked with reconstructed `StreamingChunk` objects. Overrides
+            the callback set at initialization.
+        :param generation_kwargs: Accepted for interface compatibility and ignored.
+        :param tools: Accepted for interface compatibility and ignored.
+        :param tools_strict: Accepted for interface compatibility and ignored.
+        :returns: A dictionary with a single key `replies` containing the predefined reply as a list of one
+            `ChatMessage` (empty in echo mode when there is no message to echo).
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=False
+        )
+
+        reply = self._build_reply(messages)
+        if reply is None:
+            return {"replies": []}
+
+        if streaming_callback is not None:
+            for chunk in self._make_chunks(reply):
+                streaming_callback(chunk)
+
+        return {"replies": [reply]}
+
+    @component.output_types(replies=list[ChatMessage])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,  # noqa: ARG002
+        *,
+        tools: ToolsType | None = None,  # noqa: ARG002
+        tools_strict: bool | None = None,  # noqa: ARG002
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Asynchronously return a predefined reply for the given messages without calling any API.
+
+        The signature mirrors `OpenAIChatGenerator.run_async` so the mock can be used as a positional drop-in
+        replacement.
+
+        :param messages: The conversation history as a list of `ChatMessage` instances or a single string.
+        :param streaming_callback: An optional callback invoked with reconstructed `StreamingChunk` objects. Overrides
+            the callback set at initialization.
+        :param generation_kwargs: Accepted for interface compatibility and ignored.
+        :param tools: Accepted for interface compatibility and ignored.
+        :param tools_strict: Accepted for interface compatibility and ignored.
+        :returns: A dictionary with a single key `replies` containing the predefined reply as a list of one
+            `ChatMessage` (empty in echo mode when there is no message to echo).
+        """
+        if not self._is_warmed_up:
+            self.warm_up()
+
+        messages = _normalize_messages(messages)
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=True
+        )
+
+        reply = self._build_reply(messages)
+        if reply is None:
+            return {"replies": []}
+
+        if streaming_callback is not None:
+            for chunk in self._make_chunks(reply):
+                await _invoke_streaming_callback(streaming_callback, chunk)
+
+        return {"replies": [reply]}

haystack/components/generators/chat/openai.py

@@ -0,0 +1,795 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+import os
+from datetime import datetime
+from typing import Any, ClassVar
+
+from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
+from openai.lib._pydantic import to_strict_json_schema
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    ChatCompletionMessageCustomToolCall,
+    ParsedChatCompletion,
+    ParsedChatCompletionMessage,
+)
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from pydantic import BaseModel
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.generators.utils import (
+    _convert_streaming_chunks_to_chat_message,
+    _normalize_messages,
+    _serialize_object,
+)
+from haystack.dataclasses import (
+    ChatMessage,
+    ComponentInfo,
+    FinishReason,
+    StreamingCallbackT,
+    StreamingChunk,
+    SyncStreamingCallbackT,
+    ToolCall,
+    ToolCallDelta,
+    select_streaming_callback,
+)
+from haystack.dataclasses.streaming_chunk import _invoke_streaming_callback
+from haystack.tools import (
+    ToolsType,
+    _check_duplicate_tool_names,
+    deserialize_tools_or_toolset_inplace,
+    flatten_tools_or_toolsets,
+    serialize_tools_or_toolset,
+    warm_up_tools,
+)
+from haystack.utils import Secret, deserialize_callable, serialize_callable
+from haystack.utils.http_client import init_http_client
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class OpenAIChatGenerator:
+    """
+    Completes chats using OpenAI's large language models (LLMs).
+
+    It works with the gpt-4 and gpt-5 series models and supports streaming responses
+    from OpenAI API. It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/chatmessage)
+    format in input and output.
+
+    You can customize how the text is generated by passing parameters to the
+    OpenAI API. Use the `**generation_kwargs` argument when you initialize
+    the component or when you run it. Any parameter that works with
+    `openai.ChatCompletion.create` will work here too.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat).
+
+    ### Usage example
+    ```python
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+    messages = [ChatMessage.from_user("What's Natural Language Processing?")]
+
+    client = OpenAIChatGenerator()
+    response = client.run(messages)
+    print(response)
+    ```
+    Output:
+    ```
+    {'replies':
+        [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=
+        [TextContent(text="Natural Language Processing (NLP) is a branch of artificial intelligence
+            that focuses on enabling computers to understand, interpret, and generate human language in
+            a way that is meaningful and useful.")],
+         _name=None,
+         _meta={'model': 'gpt-5-mini', 'index': 0, 'finish_reason': 'stop',
+         'usage': {'prompt_tokens': 15, 'completion_tokens': 36, 'total_tokens': 51}})
+        ]
+    }
+    ```
+    """
+
+    SUPPORTED_MODELS: ClassVar[list[str]] = [
+        "gpt-5-mini",
+        "gpt-5-nano",
+        "gpt-5",
+        "gpt-5.1",
+        "gpt-5.2",
+        "gpt-5.2-pro",
+        "gpt-5.4",
+        "gpt-5-pro",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-4.1-nano",
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4",
+        "gpt-3.5-turbo",
+    ]
+    """A non-exhaustive list of chat models supported by this component.
+    See https://developers.openai.com/api/docs/models for the full list and snapshot IDs."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
+        model: str = "gpt-5-mini",
+        streaming_callback: StreamingCallbackT | None = None,
+        api_base_url: str | None = None,
+        organization: str | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool = False,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Creates an instance of OpenAIChatGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-5-mini
+
+        Before initializing the component, you can set the 'OPENAI_TIMEOUT' and 'OPENAI_MAX_RETRIES'
+        environment variables to override the `timeout` and `max_retries` parameters respectively
+        in the OpenAI client.
+
+        :param api_key: The OpenAI API key.
+            You can set it with an environment variable `OPENAI_API_KEY`, or pass with this parameter
+            during initialization.
+        :param model: The name of the model to use.
+        :param streaming_callback: A callback function that is called when a new token is received from the stream.
+            The callback function accepts [StreamingChunk](https://docs.haystack.deepset.ai/docs/data-classes#streamingchunk)
+            as an argument.
+        :param api_base_url: An optional base URL.
+        :param organization: Your organization ID, defaults to `None`. See
+        [production best practices](https://platform.openai.com/docs/guides/production-best-practices/setting-up-your-organization).
+        :param generation_kwargs: Other parameters to use for the model. These parameters are sent directly to
+            the OpenAI endpoint. See OpenAI [documentation](https://platform.openai.com/docs/api-reference/chat) for
+            more details.
+            Some of the supported parameters:
+            - `max_completion_tokens`: An upper bound for the number of tokens that can be generated for a completion,
+                including visible output tokens and reasoning tokens.
+            - `temperature`: What sampling temperature to use. Higher values mean the model will take more risks.
+                Try 0.9 for more creative applications and 0 (argmax sampling) for ones with a well-defined answer.
+            - `top_p`: An alternative to sampling with temperature, called nucleus sampling, where the model
+                considers the results of the tokens with top_p probability mass. For example, 0.1 means only the tokens
+                comprising the top 10% probability mass are considered.
+            - `n`: How many completions to generate for each prompt. For example, if the LLM gets 3 prompts and n is 2,
+                it will generate two completions for each of the three prompts, ending up with 6 completions in total.
+            - `stop`: One or more sequences after which the LLM should stop generating tokens.
+            - `presence_penalty`: What penalty to apply if a token is already present at all. Bigger values mean
+                the model will be less likely to repeat the same token in the text.
+            - `frequency_penalty`: What penalty to apply if a token has already been generated in the text.
+                Bigger values mean the model will be less likely to repeat the same token in the text.
+            - `logit_bias`: Add a logit bias to specific tokens. The keys of the dictionary are tokens, and the
+                values are the bias to add to that token.
+            - `response_format`: A JSON schema or a Pydantic model that enforces the structure of the model's response.
+                If provided, the output will always be validated against this
+                format (unless the model returns a tool call).
+                For details, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs).
+                Notes:
+                - This parameter accepts Pydantic models and JSON schemas for latest models starting from GPT-4o.
+                  Older models only support basic version of structured outputs through `{"type": "json_object"}`.
+                  For detailed information on JSON mode, see the [OpenAI Structured Outputs documentation](https://platform.openai.com/docs/guides/structured-outputs#json-mode).
+                - For structured outputs with streaming,
+                  the `response_format` must be a JSON schema and not a Pydantic model.
+        :param timeout:
+            Timeout for OpenAI client calls. If not set, it defaults to either the
+            `OPENAI_TIMEOUT` environment variable, or 30 seconds.
+        :param max_retries:
+            Maximum number of retries to contact OpenAI after an internal error.
+            If not set, it defaults to either the `OPENAI_MAX_RETRIES` environment variable, or set to 5.
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+
+        """
+        self.api_key = api_key
+        self.model = model
+        self.generation_kwargs = generation_kwargs or {}
+        self.streaming_callback = streaming_callback
+        self.api_base_url = api_base_url
+        self.organization = organization
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.tools = tools  # Store tools as-is, whether it's a list or a Toolset
+        self.tools_strict = tools_strict
+        self.http_client_kwargs = http_client_kwargs
+        # Check for duplicate tool names
+        _check_duplicate_tool_names(flatten_tools_or_toolsets(self.tools))
+
+        self.client: OpenAI | None = None
+        self.async_client: AsyncOpenAI | None = None
+        self._tools_warmed_up = False
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_key": self.api_key.resolve_value(),
+            "organization": self.organization,
+            "base_url": self.api_base_url,
+            "timeout": timeout,
+            "max_retries": max_retries,
+        }
+
+    def _warm_up_tools(self) -> None:
+        if not self._tools_warmed_up:
+            warm_up_tools(self.tools)
+            self._tools_warmed_up = True
+
+    def warm_up(self) -> None:
+        """
+        Warm up the tools and initialize the synchronous OpenAI client.
+        """
+        self._warm_up_tools()
+        if self.client is None:
+            self.client = OpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Warm up the tools and initialize the asynchronous OpenAI client on the serving event loop.
+        """
+        self._warm_up_tools()
+        if self.async_client is None:
+            self.async_client = AsyncOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        callback_name = serialize_callable(self.streaming_callback) if self.streaming_callback else None
+        generation_kwargs = self.generation_kwargs.copy()
+        response_format = generation_kwargs.get("response_format")
+
+        # If the response format is a Pydantic model, it's converted to openai's json schema format
+        # If it's already a json schema, it's left as is
+        if response_format and isinstance(response_format, type) and issubclass(response_format, BaseModel):
+            json_schema = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": response_format.__name__,
+                    "strict": True,
+                    "schema": to_strict_json_schema(response_format),
+                },
+            }
+            generation_kwargs["response_format"] = json_schema
+
+        return default_to_dict(
+            self,
+            model=self.model,
+            streaming_callback=callback_name,
+            api_base_url=self.api_base_url,
+            organization=self.organization,
+            generation_kwargs=generation_kwargs,
+            api_key=self.api_key,
+            timeout=self.timeout,
+            max_retries=self.max_retries,
+            tools=serialize_tools_or_toolset(self.tools),
+            tools_strict=self.tools_strict,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OpenAIChatGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data: The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
+        init_params = data.get("init_parameters", {})
+        serialized_callback_handler = init_params.get("streaming_callback")
+
+        if serialized_callback_handler:
+            data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
+        return default_from_dict(cls, data)
+
+    @component.output_types(replies=list[ChatMessage])
+    def run(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        *,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Invokes chat completion based on the provided messages and generation parameters.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages. If a string is provided, it is converted
+            to a list containing a ChatMessage with user role.
+        :param streaming_callback:
+            A callback function that is called when a new token is received from the stream.
+        :param generation_kwargs:
+            Additional keyword arguments for text generation. These parameters will
+            override the parameters passed during component initialization.
+            For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
+        :param tools:
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+            If set, it will override the `tools_strict` parameter set during component initialization.
+
+        :returns:
+            A dictionary with the following key:
+            - `replies`: A list containing the generated responses as ChatMessage instances.
+        """
+        self.warm_up()
+
+        messages = _normalize_messages(messages)
+
+        if len(messages) == 0:
+            return {"replies": []}
+
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=False
+        )
+        chat_completion: Stream[ChatCompletionChunk] | ChatCompletion | ParsedChatCompletion
+
+        api_args = self._prepare_api_call(
+            messages=messages,
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            tools_strict=tools_strict,
+        )
+        openai_endpoint = api_args.pop("openai_endpoint")
+        assert self.client is not None  # mypy: client is built by warm_up above
+        openai_endpoint_method = getattr(self.client.chat.completions, openai_endpoint)
+        chat_completion = openai_endpoint_method(**api_args)
+
+        if streaming_callback is not None:
+            completions = self._handle_stream_response(
+                # we cannot check isinstance(chat_completion, Stream) because some observability tools wrap Stream
+                # and return a different type. See https://github.com/deepset-ai/haystack/issues/9014.
+                chat_completion,  # type: ignore
+                streaming_callback,
+            )
+
+        else:
+            assert isinstance(chat_completion, ChatCompletion), "Unexpected response type for non-streaming request."
+            completions = [
+                _convert_chat_completion_to_chat_message(chat_completion, choice) for choice in chat_completion.choices
+            ]
+
+        # before returning, do post-processing of the completions
+        for message in completions:
+            _check_finish_reason(message.meta)
+
+        return {"replies": completions}
+
+    @component.output_types(replies=list[ChatMessage])
+    async def run_async(
+        self,
+        messages: list[ChatMessage] | str,
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        *,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, list[ChatMessage]]:
+        """
+        Asynchronously invokes chat completion based on the provided messages and generation parameters.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in async code.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages. If a string is provided, it is converted
+            to a list containing a ChatMessage with user role.
+        :param streaming_callback:
+            A callback function that is called when a new token is received from the stream. Async callbacks are
+            preferred; a sync callback is accepted but will run synchronously on the event loop and may block it.
+        :param generation_kwargs:
+            Additional keyword arguments for text generation. These parameters will
+            override the parameters passed during component initialization.
+            For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
+        :param tools: A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
+        :param tools_strict:
+            Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
+            the schema provided in the `parameters` field of the tool definition, but this may increase latency.
+            If set, it will override the `tools_strict` parameter set during component initialization.
+
+        :returns:
+            A dictionary with the following key:
+            - `replies`: A list containing the generated responses as ChatMessage instances.
+        """
+        await self.warm_up_async()
+
+        messages = _normalize_messages(messages)
+
+        # validate and select the streaming callback
+        streaming_callback = select_streaming_callback(
+            init_callback=self.streaming_callback, runtime_callback=streaming_callback, requires_async=True
+        )
+        chat_completion: AsyncStream[ChatCompletionChunk] | ChatCompletion | ParsedChatCompletion
+
+        if len(messages) == 0:
+            return {"replies": []}
+
+        api_args = self._prepare_api_call(
+            messages=messages,
+            streaming_callback=streaming_callback,
+            generation_kwargs=generation_kwargs,
+            tools=tools,
+            tools_strict=tools_strict,
+        )
+
+        openai_endpoint = api_args.pop("openai_endpoint")
+        assert self.async_client is not None  # mypy: async_client is built by warm_up_async above
+        openai_endpoint_method = getattr(self.async_client.chat.completions, openai_endpoint)
+        chat_completion = await openai_endpoint_method(**api_args)
+
+        if streaming_callback is not None:
+            completions = await self._handle_async_stream_response(
+                # we cannot check isinstance(chat_completion, AsyncStream) because some observability tools wrap
+                # AsyncStream and return a different type. See https://github.com/deepset-ai/haystack/issues/9014.
+                chat_completion,  # type: ignore
+                streaming_callback,
+            )
+
+        else:
+            assert isinstance(chat_completion, ChatCompletion), "Unexpected response type for non-streaming request."
+            completions = [
+                _convert_chat_completion_to_chat_message(chat_completion, choice) for choice in chat_completion.choices
+            ]
+
+        # before returning, do post-processing of the completions
+        for message in completions:
+            _check_finish_reason(message.meta)
+
+        return {"replies": completions}
+
+    def _prepare_api_call(  # noqa: PLR0913
+        self,
+        *,
+        messages: list[ChatMessage],
+        streaming_callback: StreamingCallbackT | None = None,
+        generation_kwargs: dict[str, Any] | None = None,
+        tools: ToolsType | None = None,
+        tools_strict: bool | None = None,
+    ) -> dict[str, Any]:
+        # update generation kwargs by merging with the generation kwargs passed to the run method
+        generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
+
+        is_streaming = streaming_callback is not None
+        num_responses = generation_kwargs.pop("n", 1)
+
+        if is_streaming and num_responses > 1:
+            raise ValueError("Cannot stream multiple responses, please set n=1.")
+        response_format = generation_kwargs.pop("response_format", None)
+
+        # adapt ChatMessage(s) to the format expected by the OpenAI API
+        openai_formatted_messages = [message.to_openai_dict_format() for message in messages]
+
+        flattened_tools = flatten_tools_or_toolsets(tools or self.tools)
+        tools_strict = tools_strict if tools_strict is not None else self.tools_strict
+        _check_duplicate_tool_names(flattened_tools)
+
+        openai_tools = {}
+        if flattened_tools:
+            tool_definitions = []
+            for t in flattened_tools:
+                function_spec = {**t.tool_spec}
+                if tools_strict:
+                    function_spec["strict"] = True
+                    function_spec["parameters"] = _make_schema_strict(function_spec["parameters"])
+                tool_definitions.append({"type": "function", "function": function_spec})
+            openai_tools = {"tools": tool_definitions}
+
+        base_args = {
+            "model": self.model,
+            "messages": openai_formatted_messages,
+            "n": num_responses,
+            **openai_tools,
+            **generation_kwargs,
+        }
+
+        if response_format and not is_streaming:
+            # for structured outputs without streaming, we use openai's parse endpoint
+            # Note: `stream` cannot be passed to chat.completions.parse
+            # we pass a key `openai_endpoint` as a hint to the run method to use the parse endpoint
+            # this key will be removed before the API call is made
+            return {**base_args, "response_format": response_format, "openai_endpoint": "parse"}
+
+        # for structured outputs with streaming, we use openai's create endpoint
+        # we pass a key `openai_endpoint` as a hint to the run method to use the create endpoint
+        # this key will be removed before the API call is made
+        final_args = {**base_args, "stream": is_streaming, "openai_endpoint": "create"}
+
+        # We only set the response_format parameter if it's not None since None is not a valid value in the API.
+        if response_format:
+            final_args["response_format"] = response_format
+        return final_args
+
+    def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> list[ChatMessage]:
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+        for chunk in chat_completion:
+            assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
+            chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
+                chunk=chunk, previous_chunks=chunks, component_info=component_info
+            )
+            chunks.append(chunk_delta)
+            callback(chunk_delta)
+        return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
+
+    async def _handle_async_stream_response(
+        self, chat_completion: AsyncStream, callback: StreamingCallbackT
+    ) -> list[ChatMessage]:
+        component_info = ComponentInfo.from_component(self)
+        chunks: list[StreamingChunk] = []
+        try:
+            async for chunk in chat_completion:
+                assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
+                chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
+                    chunk=chunk, previous_chunks=chunks, component_info=component_info
+                )
+                chunks.append(chunk_delta)
+                await _invoke_streaming_callback(callback, chunk_delta)
+
+        except asyncio.CancelledError:
+            await asyncio.shield(chat_completion.close())
+            # close the stream when task is cancelled
+            # asyncio.shield ensures the close operation completes
+            # https://docs.python.org/3/library/asyncio-task.html#shielding-from-cancellation
+            raise  # Re-raise to propagate cancellation
+
+        return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
+
+
+def _make_schema_strict(schema: dict[str, Any]) -> dict[str, Any]:
+    """
+    Recursively transform a JSON schema to be OpenAI strict-mode compliant.
+
+    Sets `additionalProperties: false` on all objects and ensures every defined
+    property is listed in `required`. Walks into nested properties, `$defs`,
+    array `items`, and `anyOf`/`oneOf`/`allOf` combinators.
+
+    See https://platform.openai.com/docs/guides/structured-outputs#supported-schemas
+    """
+    schema = {**schema}
+
+    schema_type = schema.get("type")
+
+    if schema_type == "object" or "properties" in schema:
+        schema["additionalProperties"] = False
+        if "properties" in schema:
+            schema["required"] = list(schema["properties"].keys())
+            schema["properties"] = {k: _make_schema_strict(v) for k, v in schema["properties"].items()}
+
+    if "items" in schema:
+        schema["items"] = _make_schema_strict(schema["items"])
+
+    if "$defs" in schema:
+        schema["$defs"] = {k: _make_schema_strict(v) for k, v in schema["$defs"].items()}
+
+    for combinator in ("anyOf", "oneOf", "allOf"):
+        if combinator in schema:
+            schema[combinator] = [_make_schema_strict(s) for s in schema[combinator]]
+
+    return schema
+
+
+def _check_finish_reason(meta: dict[str, Any]) -> None:
+    if meta["finish_reason"] == "length":
+        logger.warning(
+            "The completion for index {index} has been truncated before reaching a natural stopping point. "
+            "Increase the max_completion_tokens parameter to allow for longer completions.",
+            index=meta["index"],
+            finish_reason=meta["finish_reason"],
+        )
+    if meta["finish_reason"] == "content_filter":
+        logger.warning(
+            "The completion for index {index} has been truncated due to the content filter.",
+            index=meta["index"],
+            finish_reason=meta["finish_reason"],
+        )
+
+
+def _convert_chat_completion_to_chat_message(
+    completion: ChatCompletion | ParsedChatCompletion, choice: Choice
+) -> ChatMessage:
+    """
+    Converts the non-streaming response from the OpenAI API to a ChatMessage.
+
+    :param completion: The completion returned by the OpenAI API.
+    :param choice: The choice returned by the OpenAI API.
+    :return: The ChatMessage.
+    """
+    message: ChatCompletionMessage | ParsedChatCompletionMessage = choice.message
+    text = message.content
+    tool_calls = []
+    if message.tool_calls:
+        # we currently only support function tools (not custom tools)
+        # https://platform.openai.com/docs/guides/function-calling#custom-tools
+        openai_tool_calls = [tc for tc in message.tool_calls if not isinstance(tc, ChatCompletionMessageCustomToolCall)]
+        for openai_tc in openai_tool_calls:
+            arguments_str = openai_tc.function.arguments
+            try:
+                arguments = json.loads(arguments_str)
+                tool_calls.append(ToolCall(id=openai_tc.id, tool_name=openai_tc.function.name, arguments=arguments))
+            except json.JSONDecodeError:
+                logger.warning(
+                    "OpenAI returned a malformed JSON string for tool call arguments. This tool call "
+                    "will be skipped. To always generate a valid JSON, set `tools_strict` to `True`. "
+                    "Tool call ID: {_id}, Tool name: {_name}, Arguments: {_arguments}",
+                    _id=openai_tc.id,
+                    _name=openai_tc.function.name,
+                    _arguments=arguments_str,
+                )
+
+    logprobs = _serialize_object(choice.logprobs) if choice.logprobs else None
+    meta = {
+        "model": completion.model,
+        "index": choice.index,
+        "finish_reason": choice.finish_reason,
+        "usage": _serialize_object(completion.usage),
+    }
+    if logprobs:
+        meta["logprobs"] = logprobs
+
+    return ChatMessage.from_assistant(text=text, tool_calls=tool_calls, meta=meta)
+
+
+def _convert_chat_completion_chunk_to_streaming_chunk(
+    chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: ComponentInfo | None = None
+) -> StreamingChunk:
+    """
+    Converts the streaming response chunk from the OpenAI API to a StreamingChunk.
+
+    :param chunk: The chunk returned by the OpenAI API.
+    :param previous_chunks: A list of previously received StreamingChunks.
+    :param component_info: An optional `ComponentInfo` object containing information about the component that
+        generated the chunk, such as the component name and type.
+
+    :returns:
+        A StreamingChunk object representing the content of the chunk from the OpenAI API.
+    """
+    finish_reason_mapping: dict[str, FinishReason] = {
+        "stop": "stop",
+        "length": "length",
+        "content_filter": "content_filter",
+        "tool_calls": "tool_calls",
+        "function_call": "tool_calls",
+    }
+    # On very first chunk so len(previous_chunks) == 0, the Choices field only provides role info (e.g. "assistant")
+    # Choices is empty if include_usage is set to True where the usage information is returned.
+    if len(chunk.choices) == 0:
+        return StreamingChunk(
+            content="",
+            component_info=component_info,
+            # Index is None since it's only set to an int when a content block is present
+            index=None,
+            finish_reason=None,
+            meta={
+                "model": chunk.model,
+                "received_at": datetime.now().isoformat(),
+                "usage": _serialize_object(chunk.usage),
+            },
+        )
+
+    choice: ChunkChoice = chunk.choices[0]
+
+    # create a list of ToolCallDelta objects from the tool calls
+    if choice.delta and choice.delta.tool_calls:
+        tool_calls_deltas = []
+        for tool_call in choice.delta.tool_calls:
+            function = tool_call.function
+            tool_calls_deltas.append(
+                ToolCallDelta(
+                    index=tool_call.index,
+                    id=tool_call.id,
+                    tool_name=function.name if function else None,
+                    arguments=function.arguments if function and function.arguments else None,
+                )
+            )
+        return StreamingChunk(
+            content=choice.delta.content or "",
+            component_info=component_info,
+            # We adopt the first tool_calls_deltas.index as the overall index of the chunk.
+            index=tool_calls_deltas[0].index,
+            tool_calls=tool_calls_deltas,
+            start=tool_calls_deltas[0].tool_name is not None,
+            finish_reason=finish_reason_mapping.get(choice.finish_reason) if choice.finish_reason else None,
+            meta={
+                "model": chunk.model,
+                "index": choice.index,
+                "tool_calls": choice.delta.tool_calls,
+                "finish_reason": choice.finish_reason,
+                "received_at": datetime.now().isoformat(),
+                "usage": _serialize_object(chunk.usage),
+            },
+        )
+
+    # On very first chunk the choice field only provides role info (e.g. "assistant") so we set index to None
+    # We set all chunks missing the content field to index of None. E.g. can happen if chunk only contains finish
+    # reason.
+    if choice.delta and (choice.delta.content is None or choice.delta.role is not None):
+        resolved_index = None
+    else:
+        # We set the index to be 0 since if text content is being streamed then no tool calls are being streamed
+        # NOTE: We may need to revisit this if OpenAI allows planning/thinking content before tool calls like
+        #       Anthropic Claude
+        resolved_index = 0
+
+    # Initialize meta dictionary
+    meta = {
+        "model": chunk.model,
+        "index": choice.index,
+        "tool_calls": choice.delta.tool_calls if choice.delta and choice.delta.tool_calls else None,
+        "finish_reason": choice.finish_reason,
+        "received_at": datetime.now().isoformat(),
+        "usage": _serialize_object(chunk.usage),
+    }
+
+    # check if logprobs are present
+    # logprobs are returned only for text content
+    logprobs = _serialize_object(choice.logprobs) if choice.logprobs else None
+    if logprobs:
+        meta["logprobs"] = logprobs
+
+    content = ""
+    if choice.delta and choice.delta.content:
+        content = choice.delta.content
+
+    return StreamingChunk(
+        content=content,
+        component_info=component_info,
+        index=resolved_index,
+        # The first chunk is always a start message chunk that only contains role information, so if we reach here
+        # and previous_chunks is length 1 then this is the start of text content.
+        start=len(previous_chunks) == 1,
+        finish_reason=finish_reason_mapping.get(choice.finish_reason) if choice.finish_reason else None,
+        meta=meta,
+    )

haystack/components/generators/chat/types/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .protocol import ChatGenerator
+
+__all__ = ["ChatGenerator"]

haystack/components/generators/chat/types/protocol.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Protocol
+
+from haystack.dataclasses import ChatMessage
+
+
+class ChatGenerator(Protocol):
+    """
+    Protocol for Chat Generators.
+
+    This protocol defines the minimal interface that Chat Generators must implement.
+    Chat Generators are components that process a list of `ChatMessage` objects as input and generate
+    responses using a Language Model. They return a dictionary.
+    """
+
+    def run(self, messages: list[ChatMessage]) -> dict[str, Any]:
+        """
+        Generate messages using the underlying Language Model.
+
+        Implementing classes may accept additional optional parameters in their run method.
+        For example: `def run (self, messages: list[ChatMessage], param_a="default", param_b="another_default")`.
+
+        :param messages:
+            A list of ChatMessage instances representing the input messages.
+        :returns:
+            A dictionary.
+        """
+        ...

haystack/components/generators/openai_image_generator.py

@@ -0,0 +1,246 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from typing import Any, Literal
+
+from openai import AsyncOpenAI, OpenAI
+from openai.types.image import Image
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.utils import Secret
+from haystack.utils.http_client import init_http_client
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class OpenAIImageGenerator:
+    """
+    Generates images using OpenAI's image generation models such as `gpt-image-2`.
+
+    For details on OpenAI API parameters, see
+    [OpenAI documentation](https://developers.openai.com/api/reference/resources/images/methods/generate).
+
+    ### Usage example
+    ```python
+    from haystack.components.generators import OpenAIImageGenerator
+    image_generator = OpenAIImageGenerator()
+    response = image_generator.run("Show me a picture of a black cat.")
+    print(response)
+    ```
+    """
+
+    def __init__(
+        self,
+        model: str = "gpt-image-2",
+        quality: Literal["auto", "high", "medium", "low"] = "auto",
+        size: Literal["1024x1024", "1024x1536", "1536x1024", "auto"] = "1024x1024",
+        response_format: Literal["b64_json"] = "b64_json",
+        api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
+        api_base_url: str | None = None,
+        organization: str | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        http_client_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Creates an instance of OpenAIImageGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-image-2.
+
+        :param model: The model to use for image generation. Model names can be found in the
+            [OpenAI documentation](https://developers.openai.com/api/docs/models/all).
+        :param quality: The quality of the generated image. Can be "auto", "high", "medium", or "low".
+        :param size: The size of the generated images. One of 1024x1024, 1024x1536, 1536x1024, or "auto".
+            `gpt-image-2` also supports arbitrary sizes. You can find more information about supported sizes in
+            the [OpenAI documentation](https://developers.openai.com/api/reference/resources/images/methods/generate).
+        :param response_format: This parameter is ignored and only kept for backward compatibility.
+        :param api_key: The OpenAI API key to connect to OpenAI.
+        :param api_base_url: An optional base URL.
+        :param organization: The Organization ID, defaults to `None`.
+        :param timeout:
+            Timeout for OpenAI Client calls. If not set, it is inferred from the `OPENAI_TIMEOUT` environment variable
+            or set to 30.
+        :param max_retries:
+            Maximum retries to establish contact with OpenAI if it returns an internal error. If not set, it is inferred
+            from the `OPENAI_MAX_RETRIES` environment variable or set to 5.
+        :param http_client_kwargs:
+            A dictionary of keyword arguments to configure a custom `httpx.Client`or `httpx.AsyncClient`.
+            For more information, see the [HTTPX documentation](https://www.python-httpx.org/api/#client).
+        """
+        self.model = model
+        if quality not in ["auto", "high", "medium", "low"]:
+            logger.warning("Invalid quality: {quality}. Defaulting to 'auto'.", quality=quality)
+            quality = "auto"
+        self.quality = quality
+        self.size = size
+        if response_format != "b64_json":
+            logger.warning("response_format is ignored. A base64-encoded image will be returned.")
+        self.api_key = api_key
+        self.api_base_url = api_base_url
+        self.organization = organization
+
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.http_client_kwargs = http_client_kwargs
+
+        self.client: OpenAI | None = None
+        self.async_client: AsyncOpenAI | None = None
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        timeout = self.timeout if self.timeout is not None else float(os.environ.get("OPENAI_TIMEOUT", "30.0"))
+        max_retries = (
+            self.max_retries if self.max_retries is not None else int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
+        )
+        return {
+            "api_key": self.api_key.resolve_value(),
+            "organization": self.organization,
+            "base_url": self.api_base_url,
+            "timeout": timeout,
+            "max_retries": max_retries,
+        }
+
+    def warm_up(self) -> None:
+        """
+        Initializes the synchronous OpenAI client.
+        """
+        if self.client is None:
+            self.client = OpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=False), **self._client_kwargs()
+            )
+
+    async def warm_up_async(self) -> None:  # noqa: RUF029
+        """
+        Initializes the asynchronous OpenAI client on the serving event loop.
+        """
+        if self.async_client is None:
+            self.async_client = AsyncOpenAI(
+                http_client=init_http_client(self.http_client_kwargs, async_client=True), **self._client_kwargs()
+            )
+
+    def close(self) -> None:
+        """
+        Releases the synchronous OpenAI client.
+        """
+        if self.client is not None:
+            self.client.close()
+            self.client = None
+
+    async def close_async(self) -> None:
+        """
+        Releases the asynchronous OpenAI client.
+        """
+        if self.async_client is not None:
+            await self.async_client.close()
+            self.async_client = None
+
+    @component.output_types(images=list[str], revised_prompt=str)
+    def run(
+        self,
+        prompt: str,
+        size: Literal["1024x1024", "1024x1536", "1536x1024", "auto"] | None = None,
+        quality: Literal["auto", "high", "medium", "low"] | None = None,
+        response_format: Literal["b64_json"] | None = None,  # noqa: ARG002
+    ) -> dict[str, Any]:
+        """
+        Invokes the image generation inference based on the provided prompt and generation parameters.
+
+        :param prompt: The prompt to generate the image.
+        :param size: If provided, overrides the size provided during initialization.
+        :param quality: If provided, overrides the quality provided during initialization.
+        :param response_format: This parameter is ignored and only kept for backward compatibility.
+
+        :returns:
+            A dictionary containing the generated list of images as base64 encoded JSON strings and the revised prompt.
+            The revised prompt is the prompt that was used to generate the image, if there was any revision
+            to the prompt made by OpenAI.
+        """
+        self.warm_up()
+
+        # at this point the client is initialized, but mypy doesn't know that
+        assert self.client is not None
+
+        size = size or self.size
+        quality = quality or self.quality
+        response = self.client.images.generate(model=self.model, prompt=prompt, size=size, quality=quality, n=1)
+        image_str = ""
+        revised_prompt = ""
+        if response.data is not None:
+            image: Image = response.data[0]
+            image_str = image.b64_json or ""
+            revised_prompt = image.revised_prompt or ""
+
+        return {"images": [image_str], "revised_prompt": revised_prompt}
+
+    @component.output_types(images=list[str], revised_prompt=str)
+    async def run_async(
+        self,
+        prompt: str,
+        size: Literal["1024x1024", "1024x1536", "1536x1024", "auto"] | None = None,
+        quality: Literal["auto", "high", "medium", "low"] | None = None,
+        response_format: Literal["b64_json"] | None = None,  # noqa: ARG002
+    ) -> dict[str, Any]:
+        """
+        Asynchronously invokes the image generation inference based on the provided prompt and generation parameters.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in an async code.
+
+        :param prompt: The prompt to generate the image.
+        :param size: If provided, overrides the size provided during initialization.
+        :param quality: If provided, overrides the quality provided during initialization.
+        :param response_format: This parameter is ignored and only kept for backward compatibility.
+
+        :returns:
+            A dictionary containing the generated list of images as base64 encoded JSON strings and the revised prompt.
+            The revised prompt is the prompt that was used to generate the image, if there was any revision
+            to the prompt made by OpenAI.
+        """
+        await self.warm_up_async()
+
+        # at this point the client is initialized, but mypy doesn't know that
+        assert self.async_client is not None
+
+        size = size or self.size
+        quality = quality or self.quality
+        response = await self.async_client.images.generate(
+            model=self.model, prompt=prompt, size=size, quality=quality, n=1
+        )
+        image_str = ""
+        revised_prompt = ""
+        if response.data is not None:
+            image: Image = response.data[0]
+            image_str = image.b64_json or ""
+            revised_prompt = image.revised_prompt or ""
+
+        return {"images": [image_str], "revised_prompt": revised_prompt}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        return default_to_dict(
+            self,
+            model=self.model,
+            quality=self.quality,
+            size=self.size,
+            api_key=self.api_key,
+            api_base_url=self.api_base_url,
+            organization=self.organization,
+            http_client_kwargs=self.http_client_kwargs,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "OpenAIImageGenerator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        return default_from_dict(cls, data)

haystack/components/generators/utils.py

@@ -0,0 +1,187 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from typing import Any
+
+from haystack import logging
+from haystack.dataclasses import ChatMessage, ReasoningContent, StreamingChunk, ToolCall
+
+logger = logging.getLogger(__name__)
+
+
+def print_streaming_chunk(chunk: StreamingChunk) -> None:
+    """
+    Callback function to handle and display streaming output chunks.
+
+    This function processes a `StreamingChunk` object by:
+    - Printing tool call metadata (if any), including function names and arguments, as they arrive.
+    - Printing tool call results when available.
+    - Printing the main content (e.g., text tokens) of the chunk as it is received.
+
+    The function outputs data directly to stdout and flushes output buffers to ensure immediate display during
+    streaming.
+
+    :param chunk: A chunk of streaming data containing content and optional metadata, such as tool calls and
+        tool results.
+    """
+    if chunk.start and chunk.index and chunk.index > 0:
+        # If this is the start of a new content block but not the first content block, print two new lines
+        print("\n\n", flush=True, end="")
+
+    ## Tool Call streaming
+    if chunk.tool_calls:
+        # Typically, if there are multiple tool calls in the chunk this means that the tool calls are fully formed and
+        # not just a delta.
+        for tool_call in chunk.tool_calls:
+            # If chunk.start is True indicates beginning of a tool call
+            # Also presence of tool_call.tool_name indicates the start of a tool call too
+            if chunk.start:
+                # If there is more than one tool call in the chunk, we print two new lines to separate them
+                # We know there is more than one tool call if the index of the tool call is greater than the index of
+                # the chunk.
+                if chunk.index and tool_call.index > chunk.index:
+                    print("\n\n", flush=True, end="")
+
+                print(f"[TOOL CALL]\nTool: {tool_call.tool_name} \nArguments: ", flush=True, end="")
+
+            # print the tool arguments
+            if tool_call.arguments:
+                print(tool_call.arguments, flush=True, end="")
+
+    ## Tool Call Result streaming
+    # Print tool call results if available.
+    if chunk.tool_call_result:
+        # Tool Call Result is fully formed so delta accumulation is not needed
+        print(f"[TOOL RESULT]\n{chunk.tool_call_result.result}", flush=True, end="")
+
+    ## Normal content streaming
+    # Print the main content of the chunk (from ChatGenerator)
+    if chunk.content:
+        if chunk.start:
+            print("[ASSISTANT]\n", flush=True, end="")
+        print(chunk.content, flush=True, end="")
+
+    ## Reasoning content streaming
+    # Print the reasoning content of the chunk (from ChatGenerator)
+    if chunk.reasoning:
+        if chunk.start:
+            print("[REASONING]\n", flush=True, end="")
+        print(chunk.reasoning.reasoning_text, flush=True, end="")
+
+    # End of LLM assistant message so we add two new lines
+    # This ensures spacing between multiple LLM messages (e.g. Agent) or multiple Tool Call Results
+    if chunk.finish_reason is not None:
+        print("\n\n", flush=True, end="")
+
+
+def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> ChatMessage:
+    """
+    Connects the streaming chunks into a single ChatMessage.
+
+    :param chunks: The list of all `StreamingChunk` objects.
+
+    :returns: The ChatMessage.
+    """
+    text = "".join([chunk.content for chunk in chunks])
+    logprobs = []
+    for chunk in chunks:
+        if chunk.meta.get("logprobs"):
+            logprobs.append(chunk.meta.get("logprobs"))
+    tool_calls = []
+
+    # Accumulate reasoning content from chunks
+    reasoning_parts = [chunk.reasoning.reasoning_text for chunk in chunks if chunk.reasoning]
+    reasoning = ReasoningContent(reasoning_text="".join(reasoning_parts)) if reasoning_parts else None
+
+    # Process tool calls if present in any chunk
+    tool_call_data: dict[int, dict[str, str]] = {}  # Track tool calls by index
+    for chunk in chunks:
+        if chunk.tool_calls:
+            for tool_call in chunk.tool_calls:
+                # We use the index of the tool_call to track the tool call across chunks since the ID is not always
+                # provided
+                if tool_call.index not in tool_call_data:
+                    tool_call_data[tool_call.index] = {"id": "", "name": "", "arguments": ""}
+
+                # Save the ID if present
+                if tool_call.id is not None:
+                    tool_call_data[tool_call.index]["id"] = tool_call.id
+
+                if tool_call.tool_name is not None:
+                    tool_call_data[tool_call.index]["name"] += tool_call.tool_name
+                if tool_call.arguments is not None:
+                    tool_call_data[tool_call.index]["arguments"] += tool_call.arguments
+
+    # Convert accumulated tool call data into ToolCall objects
+    sorted_keys = sorted(tool_call_data.keys())
+    for key in sorted_keys:
+        tool_call_dict = tool_call_data[key]
+        try:
+            arguments = json.loads(tool_call_dict.get("arguments", "{}")) if tool_call_dict.get("arguments") else {}
+            tool_calls.append(ToolCall(id=tool_call_dict["id"], tool_name=tool_call_dict["name"], arguments=arguments))
+        except json.JSONDecodeError:
+            logger.warning(
+                "The LLM provider returned a malformed JSON string for tool call arguments. This tool call "
+                "will be skipped. To always generate a valid JSON, set `tools_strict` to `True`. "
+                "Tool call ID: {_id}, Tool name: {_name}, Arguments: {_arguments}",
+                _id=tool_call_dict["id"],
+                _name=tool_call_dict["name"],
+                _arguments=tool_call_dict["arguments"],
+            )
+
+    # finish_reason can appear in different places so we look for the last one
+    finish_reasons = [chunk.finish_reason for chunk in chunks if chunk.finish_reason]
+    finish_reason = finish_reasons[-1] if finish_reasons else None
+
+    # usage info can appear in different chunks depending on the API provider
+    # (e.g., OpenAI returns it in the last chunk with empty choices, but Qwen3 may return it differently)
+    # so we look for the last non-None usage value across all chunks
+    usage = None
+    for chunk in reversed(chunks):
+        chunk_usage = chunk.meta.get("usage")
+        if chunk_usage is not None:
+            usage = chunk_usage
+            break
+
+    meta = {
+        "model": chunks[-1].meta.get("model"),
+        "index": 0,
+        "finish_reason": finish_reason,
+        "completion_start_time": chunks[0].meta.get("received_at"),  # first chunk received
+        "usage": usage,
+    }
+
+    if logprobs:
+        meta["logprobs"] = logprobs
+
+    return ChatMessage.from_assistant(text=text or None, tool_calls=tool_calls, reasoning=reasoning, meta=meta)
+
+
+def _serialize_object(obj: Any) -> Any:
+    """
+    Convert an object to a serializable dict recursively.
+
+    Used to serialize `logprobs` and `usage` from OpenAI SDK response objects, so it skips any
+    attribute starting with "_" (SDK-internal fields). `base_serialization._serialize_value_with_schema`
+    doesn't skip those, so don't swap this out for it.
+    """
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    if hasattr(obj, "__dict__"):
+        return {k: _serialize_object(v) for k, v in obj.__dict__.items() if not k.startswith("_")}
+    if isinstance(obj, dict):
+        return {k: _serialize_object(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_serialize_object(item) for item in obj]
+    return obj
+
+
+def _normalize_messages(messages: list[ChatMessage] | str) -> list[ChatMessage]:
+    """Normalize messages to a list of ChatMessage objects."""
+    if isinstance(messages, str):
+        return [ChatMessage.from_user(messages)]
+    if isinstance(messages, list) and all(isinstance(msg, ChatMessage) for msg in messages):
+        return messages
+    raise TypeError("Invalid messages type. Expected list[ChatMessage] or str.")

haystack/components/joiners/__init__.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "answer_joiner": ["AnswerJoiner"],
+    "branch": ["BranchJoiner"],
+    "document_joiner": ["DocumentJoiner"],
+    "list_joiner": ["ListJoiner"],
+    "string_joiner": ["StringJoiner"],
+}
+
+if TYPE_CHECKING:
+    from .answer_joiner import AnswerJoiner as AnswerJoiner
+    from .branch import BranchJoiner as BranchJoiner
+    from .document_joiner import DocumentJoiner as DocumentJoiner
+    from .list_joiner import ListJoiner as ListJoiner
+    from .string_joiner import StringJoiner as StringJoiner
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/joiners/answer_joiner.py

@@ -0,0 +1,170 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import itertools
+from collections.abc import Callable
+from enum import Enum
+from math import inf
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.core.component.types import Variadic
+from haystack.dataclasses.answer import ExtractedAnswer, GeneratedAnswer
+
+AnswerType = GeneratedAnswer | ExtractedAnswer
+
+
+class JoinMode(Enum):
+    """
+    Enum for AnswerJoiner join modes.
+    """
+
+    CONCATENATE = "concatenate"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "JoinMode":
+        """
+        Convert a string to a JoinMode enum.
+        """
+        enum_map = {e.value: e for e in JoinMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown join mode '{string}'. Supported modes in AnswerJoiner are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class AnswerJoiner:
+    """
+    Merges multiple lists of `Answer` objects into a single list.
+
+    Use this component to combine answers from different Generators into a single list.
+    Currently, the component supports only one join mode: `CONCATENATE`.
+    This mode concatenates multiple lists of answers into a single list.
+
+    ### Usage example
+
+    In this example, AnswerJoiner merges answers from two different Generators:
+
+    ```python
+    from haystack.components.builders import AnswerBuilder
+    from haystack.components.joiners import AnswerJoiner
+
+    from haystack.core.pipeline import Pipeline
+
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+
+    query = "What's Natural Language Processing?"
+    messages = [ChatMessage.from_system("You are a helpful, respectful and honest assistant. Be super concise."),
+                ChatMessage.from_user(query)]
+
+    pipe = Pipeline()
+    pipe.add_component("llm_1", OpenAIChatGenerator())
+    pipe.add_component("llm_2", OpenAIChatGenerator())
+    pipe.add_component("aba", AnswerBuilder())
+    pipe.add_component("abb", AnswerBuilder())
+    pipe.add_component("joiner", AnswerJoiner())
+
+    pipe.connect("llm_1.replies", "aba")
+    pipe.connect("llm_2.replies", "abb")
+    pipe.connect("aba.answers", "joiner")
+    pipe.connect("abb.answers", "joiner")
+
+    results = pipe.run(data={"llm_1": {"messages": messages},
+                                "llm_2": {"messages": messages},
+                                "aba": {"query": query},
+                                "abb": {"query": query}})
+    ```
+    """
+
+    def __init__(
+        self, join_mode: str | JoinMode = JoinMode.CONCATENATE, top_k: int | None = None, sort_by_score: bool = False
+    ) -> None:
+        """
+        Creates an AnswerJoiner component.
+
+        :param join_mode:
+            Specifies the join mode to use. Available modes:
+            - `concatenate`: Concatenates multiple lists of Answers into a single list.
+        :param top_k:
+            The maximum number of Answers to return.
+        :param sort_by_score:
+            If `True`, sorts the documents by score in descending order.
+            If a document has no score, it is handled as if its score is -infinity.
+        """
+        if isinstance(join_mode, str):
+            join_mode = JoinMode.from_str(join_mode)
+        join_mode_functions: dict[JoinMode, Callable[[list[list[AnswerType]]], list[AnswerType]]] = {
+            JoinMode.CONCATENATE: self._concatenate
+        }
+        self.join_mode_function: Callable[[list[list[AnswerType]]], list[AnswerType]] = join_mode_functions[join_mode]
+        self.join_mode = join_mode
+        self.top_k = top_k
+        self.sort_by_score = sort_by_score
+
+    @component.output_types(answers=list[AnswerType])
+    def run(self, answers: Variadic[list[AnswerType]], top_k: int | None = None) -> dict[str, Any]:
+        """
+        Joins multiple lists of Answers into a single list depending on the `join_mode` parameter.
+
+        :param answers:
+            Nested list of Answers to be merged.
+
+        :param top_k:
+            The maximum number of Answers to return. Overrides the instance's `top_k` if provided.
+
+        :returns:
+            A dictionary with the following keys:
+            - `answers`: Merged list of Answers
+        """
+        answers_list = list(answers)
+        join_function = self.join_mode_function
+        output_answers: list[AnswerType] = join_function(answers_list)
+
+        if self.sort_by_score:
+            output_answers = sorted(
+                output_answers,
+                key=lambda answer: score if (score := getattr(answer, "score", None)) is not None else -inf,
+                reverse=True,
+            )
+
+        top_k = top_k or self.top_k
+        if top_k:
+            output_answers = output_answers[:top_k]
+        return {"answers": output_answers}
+
+    def _concatenate(self, answer_lists: list[list[AnswerType]]) -> list[AnswerType]:
+        """
+        Concatenate multiple lists of Answers, flattening them into a single list and sorting by score.
+
+        :param answer_lists: List of lists of Answers to be flattened.
+        """
+        return list(itertools.chain.from_iterable(answer_lists))
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, join_mode=str(self.join_mode), top_k=self.top_k, sort_by_score=self.sort_by_score)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AnswerJoiner":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        return default_from_dict(cls, data)

haystack/components/joiners/branch.py

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.core.component.types import GreedyVariadic
+from haystack.utils import deserialize_type, serialize_type
+
+
+@component
+class BranchJoiner:
+    """
+    A component that merges multiple input branches of a pipeline into a single output stream.
+
+    `BranchJoiner` receives multiple inputs of the same data type and forwards the first received value
+    to its output. This is useful for scenarios where multiple branches need to converge before proceeding.
+
+    ### Common Use Cases:
+    - **Loop Handling:** `BranchJoiner` helps close loops in pipelines. For example, if a pipeline component validates
+      or modifies incoming data and produces an error-handling branch, `BranchJoiner` can merge both branches and send
+      (or resend in the case of a loop) the data to the component that evaluates errors. See "Usage example" below.
+
+    - **Decision-Based Merging:** `BranchJoiner` reconciles branches coming from Router components (such as
+      `ConditionalRouter`, `TextLanguageRouter`). Suppose a `TextLanguageRouter` directs user queries to different
+      Retrievers based on the detected language. Each Retriever processes its assigned query and passes the results
+      to `BranchJoiner`, which consolidates them into a single output before passing them to the next component, such
+      as a `PromptBuilder`.
+
+    ### Example Usage:
+    ```python
+    import json
+
+    from haystack import Pipeline
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.joiners import BranchJoiner
+    from haystack.components.validators import JsonSchemaValidator
+    from haystack.dataclasses import ChatMessage
+
+    # Define a schema for validation
+    person_schema = {
+        "type": "object",
+        "properties": {
+            "first_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
+            "last_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
+            "nationality": {"type": "string", "enum": ["Italian", "Portuguese", "American"]},
+        },
+        "required": ["first_name", "last_name", "nationality"]
+    }
+
+    # Initialize a pipeline
+    pipe = Pipeline()
+
+    # Add components to the pipeline
+    pipe.add_component("joiner", BranchJoiner(list[ChatMessage]))
+    pipe.add_component("generator", OpenAIChatGenerator(model="gpt-4.1-mini"))
+    pipe.add_component("validator", JsonSchemaValidator(json_schema=person_schema))
+
+    # And connect them
+    pipe.connect("joiner", "generator")
+    pipe.connect("generator.replies", "validator.messages")
+    pipe.connect("validator.validation_error", "joiner")
+
+    result = pipe.run(
+        data={
+        "generator": {"generation_kwargs": {"response_format": {"type": "json_object"}}},
+        "joiner": {"value": [ChatMessage.from_user("Create json from Peter Parker")]}}
+    )
+
+    print(json.loads(result["validator"]["validated"][0].text))
+
+
+    # >> {'first_name': 'Peter', 'last_name': 'Parker', 'nationality': 'American', 'name': 'Spider-Man', 'occupation':
+    # >> 'Superhero', 'age': 23, 'location': 'New York City'}
+    ```
+
+    Note that `BranchJoiner` can manage only one data type at a time. In this case, `BranchJoiner` is created for
+    passing `list[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
+    connected components and also the type of data that `BranchJoiner` will send through its output.
+
+    In the code example, `BranchJoiner` receives a looped back `list[ChatMessage]` from the `JsonSchemaValidator` and
+    sends it down to the `OpenAIChatGenerator` for re-generation. We can have multiple loopback connections in the
+    pipeline. In this instance, the downstream component is only one (the `OpenAIChatGenerator`), but the pipeline could
+    have more than one downstream component.
+    """
+
+    def __init__(self, type_: type) -> None:
+        """
+        Creates a `BranchJoiner` component.
+
+        :param type_: The expected data type of inputs and outputs.
+        """
+        self.type_ = type_
+        component.set_input_types(self, value=GreedyVariadic[type_])  # type: ignore
+        component.set_output_types(self, value=type_)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component into a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, type_=serialize_type(self.type_))
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "BranchJoiner":
+        """
+        Deserializes a `BranchJoiner` instance from a dictionary.
+
+        :param data: The dictionary containing serialized component data.
+        :returns:
+            A deserialized `BranchJoiner` instance.
+        """
+        data["init_parameters"]["type_"] = deserialize_type(data["init_parameters"]["type_"])
+        return default_from_dict(cls, data)
+
+    def run(self, **kwargs: Any) -> dict[str, Any]:
+        """
+        Executes the `BranchJoiner`, selecting the first available input value and passing it downstream.
+
+        :param **kwargs: The input data. Must be of the type declared by `type_` during initialization.
+        :returns:
+            A dictionary with a single key `value`, containing the first input received.
+        """
+        if (inputs_count := len(kwargs["value"])) != 1:
+            raise ValueError(f"BranchJoiner expects only one input, but {inputs_count} were received.")
+        return {"value": kwargs["value"][0]}

haystack/components/joiners/document_joiner.py

@@ -0,0 +1,271 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import itertools
+from collections import defaultdict
+from dataclasses import replace
+from enum import Enum
+from math import inf
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.core.component.types import Variadic
+from haystack.utils.misc import _reciprocal_rank_fusion
+
+logger = logging.getLogger(__name__)
+
+
+class JoinMode(Enum):
+    """
+    Enum for join mode.
+    """
+
+    CONCATENATE = "concatenate"
+    MERGE = "merge"
+    RECIPROCAL_RANK_FUSION = "reciprocal_rank_fusion"
+    DISTRIBUTION_BASED_RANK_FUSION = "distribution_based_rank_fusion"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "JoinMode":
+        """
+        Convert a string to a JoinMode enum.
+        """
+        enum_map = {e.value: e for e in JoinMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown join mode '{string}'. Supported modes in DocumentJoiner are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class DocumentJoiner:
+    """
+    Joins multiple lists of documents into a single list.
+
+    It supports different join modes:
+    - concatenate: Keeps the highest-scored document in case of duplicates.
+    - merge: Calculates a weighted sum of scores for duplicates and merges them.
+    - reciprocal_rank_fusion: Merges and assigns scores based on reciprocal rank fusion.
+    - distribution_based_rank_fusion: Merges and assigns scores based on scores distribution in each Retriever.
+
+    ### Usage example:
+
+    ```python
+    from haystack import Pipeline, Document
+    from haystack.components.embedders import OpenAITextEmbedder, OpenAIDocumentEmbedder
+    from haystack.components.joiners import DocumentJoiner
+    from haystack.components.retrievers import InMemoryBM25Retriever
+    from haystack.components.retrievers import InMemoryEmbeddingRetriever
+    from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+    document_store = InMemoryDocumentStore()
+    docs = [Document(content="Paris"), Document(content="Berlin"), Document(content="London")]
+    embedder = OpenAIDocumentEmbedder()
+    docs_embeddings = embedder.run(docs)
+    document_store.write_documents(docs_embeddings['documents'])
+
+    p = Pipeline()
+    p.add_component(instance=InMemoryBM25Retriever(document_store=document_store), name="bm25_retriever")
+    p.add_component(
+            instance=OpenAITextEmbedder(),
+            name="text_embedder",
+        )
+    p.add_component(instance=InMemoryEmbeddingRetriever(document_store=document_store), name="embedding_retriever")
+    p.add_component(instance=DocumentJoiner(), name="joiner")
+    p.connect("bm25_retriever", "joiner")
+    p.connect("embedding_retriever", "joiner")
+    p.connect("text_embedder", "embedding_retriever")
+    query = "What is the capital of France?"
+    p.run(data={"query": query, "text": query, "top_k": 1})
+    ```
+    """
+
+    def __init__(
+        self,
+        join_mode: str | JoinMode = JoinMode.CONCATENATE,
+        weights: list[float] | None = None,
+        top_k: int | None = None,
+        sort_by_score: bool = True,
+    ) -> None:
+        """
+        Creates a DocumentJoiner component.
+
+        :param join_mode:
+            Specifies the join mode to use. Available modes:
+            - `concatenate`: Keeps the highest-scored document in case of duplicates.
+            - `merge`: Calculates a weighted sum of scores for duplicates and merges them.
+            - `reciprocal_rank_fusion`: Merges and assigns scores based on reciprocal rank fusion.
+            - `distribution_based_rank_fusion`: Merges and assigns scores based on scores
+            distribution in each Retriever.
+        :param weights:
+            Assign importance to each list of documents to influence how they're joined.
+            This parameter is ignored for
+            `concatenate` or `distribution_based_rank_fusion` join modes.
+            Weight for each list of documents must match the number of inputs.
+        :param top_k:
+            The maximum number of documents to return.
+        :param sort_by_score:
+            If `True`, sorts the documents by score in descending order.
+            If a document has no score, it is handled as if its score is -infinity.
+        """
+        if isinstance(join_mode, str):
+            join_mode = JoinMode.from_str(join_mode)
+        join_mode_functions = {
+            JoinMode.CONCATENATE: DocumentJoiner._concatenate,
+            JoinMode.MERGE: self._merge,
+            JoinMode.RECIPROCAL_RANK_FUSION: self._rrf,
+            JoinMode.DISTRIBUTION_BASED_RANK_FUSION: DocumentJoiner._distribution_based_rank_fusion,
+        }
+        self.join_mode_function = join_mode_functions[join_mode]
+        self.join_mode = join_mode
+        if weights:
+            weight_sum = sum(weights)
+            if weight_sum == 0:
+                raise ValueError("The provided `weights` must not sum to zero.")
+            self.weights: list[float] | None = [float(i) / weight_sum for i in weights]
+        else:
+            self.weights = None
+        self.top_k = top_k
+        self.sort_by_score = sort_by_score
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: Variadic[list[Document]], top_k: int | None = None) -> dict[str, Any]:
+        """
+        Joins multiple lists of Documents into a single list depending on the `join_mode` parameter.
+
+        :param documents:
+            List of list of documents to be merged.
+        :param top_k:
+            The maximum number of documents to return. Overrides the instance's `top_k` if provided.
+
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Merged list of Documents
+        """
+        documents = list(documents)
+        output_documents = self.join_mode_function(documents)
+
+        if self.sort_by_score:
+            output_documents = sorted(
+                output_documents, key=lambda doc: doc.score if doc.score is not None else -inf, reverse=True
+            )
+            if any(doc.score is None for doc in output_documents):
+                logger.info(
+                    "Some of the Documents DocumentJoiner got have score=None. It was configured to sort Documents by "
+                    "score, so those with score=None were sorted as if they had a score of -infinity."
+                )
+
+        if top_k:
+            output_documents = output_documents[:top_k]
+        elif self.top_k:
+            output_documents = output_documents[: self.top_k]
+
+        return {"documents": output_documents}
+
+    @staticmethod
+    def _concatenate(document_lists: list[list[Document]]) -> list[Document]:
+        """
+        Concatenate multiple lists of Documents and return only the Document with the highest score for duplicates.
+        """
+        output = []
+        docs_per_id = defaultdict(list)
+        for doc in itertools.chain.from_iterable(document_lists):
+            docs_per_id[doc.id].append(doc)
+        for docs in docs_per_id.values():
+            doc_with_best_score = max(docs, key=lambda doc: doc.score if doc.score is not None else -inf)
+            output.append(doc_with_best_score)
+        return output
+
+    def _merge(self, document_lists: list[list[Document]]) -> list[Document]:
+        """
+        Merge multiple lists of Documents and calculate a weighted sum of the scores of duplicate Documents.
+        """
+        # This check prevents a division by zero when no documents are passed
+        if not document_lists:
+            return []
+
+        scores_map: dict = defaultdict(int)
+        documents_map = {}
+        weights = self.weights if self.weights else [1 / len(document_lists)] * len(document_lists)
+
+        for documents, weight in zip(document_lists, weights, strict=True):
+            for doc in documents:
+                scores_map[doc.id] += (doc.score if doc.score is not None else 0) * weight
+                documents_map[doc.id] = doc
+
+        return [replace(doc, score=scores_map[doc.id]) for doc in documents_map.values()]
+
+    def _rrf(self, document_lists: list[list[Document]]) -> list[Document]:
+        """
+        Merge multiple lists of Documents and assign scores based on reciprocal rank fusion.
+        """
+        return _reciprocal_rank_fusion(document_lists, weights=self.weights)
+
+    @staticmethod
+    def _distribution_based_rank_fusion(document_lists: list[list[Document]]) -> list[Document]:
+        """
+        Merge multiple lists of Documents and assign scores based on Distribution-Based Score Fusion.
+
+        (https://medium.com/plain-simple-software/distribution-based-score-fusion-dbsf-a-new-approach-to-vector-search-ranking-f87c37488b18)
+        If a Document is in more than one retriever, the one with the highest score is used.
+        """
+        rescaled_lists: list[list[Document]] = []
+        for documents in document_lists:
+            if len(documents) == 0:
+                rescaled_lists.append(documents)
+                continue
+
+            scores_list = [doc.score if doc.score is not None else 0 for doc in documents]
+
+            mean_score = sum(scores_list) / len(scores_list)
+            std_dev = (sum((x - mean_score) ** 2 for x in scores_list) / len(scores_list)) ** 0.5
+            min_score = mean_score - 3 * std_dev
+            max_score = mean_score + 3 * std_dev
+            delta_score = max_score - min_score
+
+            # if all docs have the same score delta_score is 0, the docs are uninformative for the query
+            rescaled_lists.append(
+                [
+                    replace(
+                        doc,
+                        score=((doc.score if doc.score is not None else 0) - min_score) / delta_score
+                        if delta_score != 0.0
+                        else 0.0,
+                    )
+                    for doc in documents
+                ]
+            )
+
+        return DocumentJoiner._concatenate(document_lists=rescaled_lists)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            join_mode=str(self.join_mode),
+            weights=self.weights,
+            top_k=self.top_k,
+            sort_by_score=self.sort_by_score,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "DocumentJoiner":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        return default_from_dict(cls, data)

haystack/components/joiners/list_joiner.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from itertools import chain
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.core.component.types import Variadic
+from haystack.utils import deserialize_type, serialize_type
+
+
+@component
+class ListJoiner:
+    """
+    A component that joins multiple lists into a single flat list.
+
+    The ListJoiner receives multiple lists of the same type and concatenates them into a single flat list.
+    The output order respects the pipeline's execution sequence, with earlier inputs being added first.
+
+    Usage example:
+    ```python
+    from haystack.components.builders import ChatPromptBuilder
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+    from haystack import Pipeline
+    from haystack.components.joiners import ListJoiner
+
+
+    user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")]
+
+    feedback_prompt = \"""
+        You are given a question and an answer.
+        Your task is to provide a score and a brief feedback on the answer.
+        Question: {{query}}
+        Answer: {{response}}
+        \"""
+    feedback_message = [ChatMessage.from_system(feedback_prompt)]
+
+    prompt_builder = ChatPromptBuilder(template=user_message)
+    feedback_prompt_builder = ChatPromptBuilder(template=feedback_message)
+    llm = OpenAIChatGenerator()
+    feedback_llm = OpenAIChatGenerator()
+
+    pipe = Pipeline()
+    pipe.add_component("prompt_builder", prompt_builder)
+    pipe.add_component("llm", llm)
+    pipe.add_component("feedback_prompt_builder", feedback_prompt_builder)
+    pipe.add_component("feedback_llm", feedback_llm)
+    pipe.add_component("list_joiner", ListJoiner(list[ChatMessage]))
+
+    pipe.connect("prompt_builder.prompt", "llm.messages")
+    pipe.connect("prompt_builder.prompt", "list_joiner")
+    pipe.connect("llm.replies", "list_joiner")
+    pipe.connect("llm.replies", "feedback_prompt_builder.response")
+    pipe.connect("feedback_prompt_builder.prompt", "feedback_llm.messages")
+    pipe.connect("feedback_llm.replies", "list_joiner")
+
+    query = "What is nuclear physics?"
+    ans = pipe.run(data={"prompt_builder": {"template_variables":{"query": query}},
+        "feedback_prompt_builder": {"template_variables":{"query": query}}})
+
+    print(ans["list_joiner"]["values"])
+    ```
+    """
+
+    def __init__(self, list_type_: type | None = None) -> None:
+        """
+        Creates a ListJoiner component.
+
+        :param list_type_: The expected type of the lists this component will join (e.g., list[ChatMessage]).
+            If specified, all input lists must conform to this type. If None, the component defaults to handling
+            lists of any type including mixed types.
+        """
+        self.list_type_ = list_type_
+        if list_type_ is not None:
+            component.set_output_types(self, values=list_type_)
+        else:
+            component.set_output_types(self, values=list[Any])
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self, list_type_=serialize_type(self.list_type_) if self.list_type_ is not None else None
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ListJoiner":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :returns: Deserialized component.
+        """
+        init_parameters = data.get("init_parameters")
+        if init_parameters is not None and init_parameters.get("list_type_") is not None:
+            data["init_parameters"]["list_type_"] = deserialize_type(data["init_parameters"]["list_type_"])
+        return default_from_dict(cls, data)
+
+    def run(self, values: Variadic[list[Any]]) -> dict[str, list[Any]]:
+        """
+        Joins multiple lists into a single flat list.
+
+        :param values: The list to be joined.
+        :returns: Dictionary with 'values' key containing the joined list.
+        """
+        result = list(chain(*values))
+        return {"values": result}

haystack/components/joiners/string_joiner.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+from haystack import component
+from haystack.core.component.types import Variadic
+
+
+@component
+class StringJoiner:
+    """
+    Component to join strings from different components to a list of strings.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.joiners import StringJoiner
+    from haystack.components.builders import PromptBuilder
+    from haystack.core.pipeline import Pipeline
+
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+
+    string_1 = "What's Natural Language Processing?"
+    string_2 = "What is life?"
+
+    pipeline = Pipeline()
+    pipeline.add_component("prompt_builder_1", PromptBuilder("Builder 1: {{query}}"))
+    pipeline.add_component("prompt_builder_2", PromptBuilder("Builder 2: {{query}}"))
+    pipeline.add_component("string_joiner", StringJoiner())
+
+    pipeline.connect("prompt_builder_1.prompt", "string_joiner.strings")
+    pipeline.connect("prompt_builder_2.prompt", "string_joiner.strings")
+
+    print(pipeline.run(data={"prompt_builder_1": {"query": string_1}, "prompt_builder_2": {"query": string_2}}))
+
+    # >> {"string_joiner": {"strings": ["Builder 1: What's Natural Language Processing?", "Builder 2: What is life?"]}}
+    ```
+    """
+
+    @component.output_types(strings=list[str])
+    def run(self, strings: Variadic[str]) -> dict[str, list[str]]:
+        """
+        Joins strings into a list of strings
+
+        :param strings:
+            strings from different components
+
+        :returns:
+            A dictionary with the following keys:
+            - `strings`: Merged list of strings
+        """
+
+        out_strings = list(strings)
+        return {"strings": out_strings}

haystack/components/preprocessors/__init__.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "csv_document_cleaner": ["CSVDocumentCleaner"],
+    "csv_document_splitter": ["CSVDocumentSplitter"],
+    "document_cleaner": ["DocumentCleaner"],
+    "document_preprocessor": ["DocumentPreprocessor"],
+    "document_splitter": ["DocumentSplitter"],
+    "embedding_based_document_splitter": ["EmbeddingBasedDocumentSplitter"],
+    "hierarchical_document_splitter": ["HierarchicalDocumentSplitter"],
+    "markdown_header_splitter": ["MarkdownHeaderSplitter"],
+    "python_code_splitter": ["PythonCodeSplitter"],
+    "recursive_splitter": ["RecursiveDocumentSplitter"],
+    "text_cleaner": ["TextCleaner"],
+}
+
+if TYPE_CHECKING:
+    from .csv_document_cleaner import CSVDocumentCleaner as CSVDocumentCleaner
+    from .csv_document_splitter import CSVDocumentSplitter as CSVDocumentSplitter
+    from .document_cleaner import DocumentCleaner as DocumentCleaner
+    from .document_preprocessor import DocumentPreprocessor as DocumentPreprocessor
+    from .document_splitter import DocumentSplitter as DocumentSplitter
+    from .embedding_based_document_splitter import EmbeddingBasedDocumentSplitter as EmbeddingBasedDocumentSplitter
+    from .hierarchical_document_splitter import HierarchicalDocumentSplitter as HierarchicalDocumentSplitter
+    from .markdown_header_splitter import MarkdownHeaderSplitter as MarkdownHeaderSplitter
+    from .python_code_splitter import PythonCodeSplitter as PythonCodeSplitter
+    from .recursive_splitter import RecursiveDocumentSplitter as RecursiveDocumentSplitter
+    from .text_cleaner import TextCleaner as TextCleaner
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/preprocessors/csv_document_cleaner.py

@@ -0,0 +1,178 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from copy import deepcopy
+from io import StringIO
+from typing import Optional
+
+from haystack import Document, component, logging
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install pandas'") as pandas_import:
+    import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class CSVDocumentCleaner:
+    """
+    A component for cleaning CSV documents by removing empty rows and columns.
+
+    This component processes CSV content stored in Documents, allowing
+    for the optional ignoring of a specified number of rows and columns before performing
+    the cleaning operation. Additionally, it provides options to keep document IDs and
+    control whether empty rows and columns should be removed.
+    """
+
+    def __init__(
+        self,
+        *,
+        ignore_rows: int = 0,
+        ignore_columns: int = 0,
+        remove_empty_rows: bool = True,
+        remove_empty_columns: bool = True,
+        keep_id: bool = False,
+    ) -> None:
+        """
+        Initializes the CSVDocumentCleaner component.
+
+        :param ignore_rows: Number of rows to ignore from the top of the CSV table before processing.
+        :param ignore_columns: Number of columns to ignore from the left of the CSV table before processing.
+        :param remove_empty_rows: Whether to remove rows that are entirely empty.
+        :param remove_empty_columns: Whether to remove columns that are entirely empty.
+        :param keep_id: Whether to retain the original document ID in the output document.
+
+        Rows and columns ignored using these parameters are preserved in the final output, meaning
+        they are not considered when removing empty rows and columns.
+        """
+        self.ignore_rows = ignore_rows
+        self.ignore_columns = ignore_columns
+        self.remove_empty_rows = remove_empty_rows
+        self.remove_empty_columns = remove_empty_columns
+        self.keep_id = keep_id
+        pandas_import.check()
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Cleans CSV documents by removing empty rows and columns while preserving specified ignored rows and columns.
+
+        :param documents: List of Documents containing CSV-formatted content.
+        :return: A dictionary with a list of cleaned Documents under the key "documents".
+
+        Processing steps:
+        1. Reads each document's content as a CSV table.
+        2. Retains the specified number of `ignore_rows` from the top and `ignore_columns` from the left.
+        3. Drops any rows and columns that are entirely empty (if enabled by `remove_empty_rows` and
+            `remove_empty_columns`).
+        4. Reattaches the ignored rows and columns to maintain their original positions.
+        5. Returns the cleaned CSV content as a new `Document` object, with an option to retain the original
+            document ID.
+        """
+        if len(documents) == 0:
+            return {"documents": []}
+
+        ignore_rows = self.ignore_rows
+        ignore_columns = self.ignore_columns
+
+        cleaned_documents = []
+        for document in documents:
+            try:
+                df = pd.read_csv(StringIO(document.content), header=None, dtype=object)
+            except Exception as e:
+                logger.exception(
+                    "Error processing document {id}. Keeping it, but skipping cleaning. Error: {error}",
+                    id=document.id,
+                    error=e,
+                )
+                cleaned_documents.append(document)
+                continue
+
+            if ignore_rows > df.shape[0] or ignore_columns > df.shape[1]:
+                logger.warning(
+                    "Document {id} has fewer rows {df_rows} or columns {df_cols} "
+                    "than the number of rows {rows} or columns {cols} to ignore. "
+                    "Keeping the entire document.",
+                    id=document.id,
+                    df_rows=df.shape[0],
+                    df_cols=df.shape[1],
+                    rows=ignore_rows,
+                    cols=ignore_columns,
+                )
+                cleaned_documents.append(document)
+                continue
+
+            final_df = self._clean_df(df=df, ignore_rows=ignore_rows, ignore_columns=ignore_columns)
+
+            clean_doc = Document(
+                id=document.id if self.keep_id else "",
+                content=final_df.to_csv(index=False, header=False, lineterminator="\n"),
+                blob=document.blob,
+                meta=deepcopy(document.meta),
+                score=document.score,
+                embedding=document.embedding,
+                sparse_embedding=document.sparse_embedding,
+            )
+            cleaned_documents.append(clean_doc)
+        return {"documents": cleaned_documents}
+
+    def _clean_df(self, df: "pd.DataFrame", ignore_rows: int, ignore_columns: int) -> "pd.DataFrame":
+        """
+        Cleans a DataFrame by removing empty rows and columns while preserving ignored sections.
+
+        :param df: The input DataFrame representing the CSV data.
+        :param ignore_rows: Number of top rows to ignore.
+        :param ignore_columns: Number of left columns to ignore.
+        """
+        # Get ignored rows and columns
+        ignored_rows = self._get_ignored_rows(df=df, ignore_rows=ignore_rows)
+        ignored_columns = self._get_ignored_columns(df=df, ignore_columns=ignore_columns)
+        final_df = df.iloc[ignore_rows:, ignore_columns:]
+
+        # Drop rows that are entirely empty
+        if self.remove_empty_rows:
+            final_df = final_df.dropna(axis=0, how="all")
+
+        # Drop columns that are entirely empty
+        if self.remove_empty_columns:
+            final_df = final_df.dropna(axis=1, how="all")
+
+        # Reattach ignored rows
+        if ignore_rows > 0 and ignored_rows is not None:
+            # Keep only relevant columns
+            ignored_rows = ignored_rows.loc[:, final_df.columns]
+            final_df = pd.concat([ignored_rows, final_df], axis=0)
+
+        # Reattach ignored columns
+        if ignore_columns > 0 and ignored_columns is not None:
+            # Keep only relevant rows
+            ignored_columns = ignored_columns.loc[final_df.index, :]
+            final_df = pd.concat([ignored_columns, final_df], axis=1)
+
+        return final_df
+
+    @staticmethod
+    def _get_ignored_rows(df: "pd.DataFrame", ignore_rows: int) -> Optional["pd.DataFrame"]:
+        """
+        Extracts the rows to be ignored from the DataFrame.
+
+        :param df: The input DataFrame.
+        :param ignore_rows: Number of rows to extract from the top.
+        """
+        if ignore_rows > 0:
+            return df.iloc[:ignore_rows, :]
+        return None
+
+    @staticmethod
+    def _get_ignored_columns(df: "pd.DataFrame", ignore_columns: int) -> Optional["pd.DataFrame"]:
+        """
+        Extracts the columns to be ignored from the DataFrame.
+
+        :param df: The input DataFrame.
+        :param ignore_columns: Number of columns to extract from the left.
+        """
+        if ignore_columns > 0:
+            return df.iloc[:, :ignore_columns]
+        return None

haystack/components/preprocessors/csv_document_splitter.py

@@ -0,0 +1,286 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from io import StringIO
+from typing import Any, Literal, get_args
+
+from haystack import Document, component, logging
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install pandas'") as pandas_import:
+    import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+SplitMode = Literal["threshold", "row-wise"]
+
+
+@component
+class CSVDocumentSplitter:
+    """
+    A component for splitting CSV documents into sub-tables based on split arguments.
+
+    The splitter supports two modes of operation:
+    - identify consecutive empty rows or columns that exceed a given threshold
+    and uses them as delimiters to segment the document into smaller tables.
+    - split each row into a separate sub-table, represented as a Document.
+
+    """
+
+    def __init__(
+        self,
+        row_split_threshold: int | None = 2,
+        column_split_threshold: int | None = 2,
+        read_csv_kwargs: dict[str, Any] | None = None,
+        split_mode: SplitMode = "threshold",
+    ) -> None:
+        """
+        Initializes the CSVDocumentSplitter component.
+
+        :param row_split_threshold: The minimum number of consecutive empty rows required to trigger a split.
+        :param column_split_threshold: The minimum number of consecutive empty columns required to trigger a split.
+        :param read_csv_kwargs: Additional keyword arguments to pass to `pandas.read_csv`.
+            By default, the component with options:
+            - `header=None`
+            - `skip_blank_lines=False` to preserve blank lines
+            - `dtype=object` to prevent type inference (e.g., converting numbers to floats).
+            See https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html for more information.
+        :param split_mode:
+            If `threshold`, the component will split the document based on the number of
+            consecutive empty rows or columns that exceed the `row_split_threshold` or `column_split_threshold`.
+            If `row-wise`, the component will split each row into a separate sub-table.
+        """
+        pandas_import.check()
+        if split_mode not in get_args(SplitMode):
+            raise ValueError(
+                f"Split mode '{split_mode}' not recognized. Choose one among: {', '.join(get_args(SplitMode))}."
+            )
+        if row_split_threshold is not None and row_split_threshold < 1:
+            raise ValueError("row_split_threshold must be greater than 0")
+
+        if column_split_threshold is not None and column_split_threshold < 1:
+            raise ValueError("column_split_threshold must be greater than 0")
+
+        if row_split_threshold is None and column_split_threshold is None:
+            raise ValueError("At least one of row_split_threshold or column_split_threshold must be specified.")
+
+        self.row_split_threshold = row_split_threshold
+        self.column_split_threshold = column_split_threshold
+        self.read_csv_kwargs = read_csv_kwargs or {}
+        self.split_mode = split_mode
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Processes and splits a list of CSV documents into multiple sub-tables.
+
+        **Splitting Process:**
+        1. Applies a row-based split if `row_split_threshold` is provided.
+        2. Applies a column-based split if `column_split_threshold` is provided.
+        3. If both thresholds are specified, performs a recursive split by rows first, then columns, ensuring
+           further fragmentation of any sub-tables that still contain empty sections.
+        4. Sorts the resulting sub-tables based on their original positions within the document.
+
+        :param documents: A list of Documents containing CSV-formatted content.
+            Each document is assumed to contain one or more tables separated by empty rows or columns.
+
+        :return:
+            A dictionary with a key `"documents"`, mapping to a list of new `Document` objects,
+            each representing an extracted sub-table from the original CSV.
+            The metadata of each document includes:
+                - A field `source_id` to track the original document.
+                - A field `row_idx_start` to indicate the starting row index of the sub-table in the original table.
+                - A field `col_idx_start` to indicate the starting column index of the sub-table in the original table.
+                - A field `split_id` to indicate the order of the split in the original document.
+                - All other metadata copied from the original document.
+
+        - If a document cannot be processed, it is returned unchanged.
+        - The `meta` field from the original document is preserved in the split documents.
+        """
+        if len(documents) == 0:
+            return {"documents": documents}
+
+        resolved_read_csv_kwargs = {"header": None, "skip_blank_lines": False, "dtype": object, **self.read_csv_kwargs}
+
+        split_documents = []
+        split_dfs = []
+        for document in documents:
+            try:
+                df = pd.read_csv(StringIO(document.content), **resolved_read_csv_kwargs)
+            except Exception as e:
+                logger.exception(
+                    "Error processing document {document_id}. Keeping it, but skipping splitting. Error: {error}",
+                    document_id=document.id,
+                    error=e,
+                )
+                split_documents.append(document)
+                continue
+
+            if self.split_mode == "row-wise":
+                # each row is a separate sub-table
+                split_dfs = self._split_by_row(df=df)
+
+            elif self.split_mode == "threshold":
+                if self.row_split_threshold is not None and self.column_split_threshold is None:
+                    # split by rows
+                    split_dfs = self._split_dataframe(df=df, split_threshold=self.row_split_threshold, axis="row")
+                elif self.column_split_threshold is not None and self.row_split_threshold is None:
+                    # split by columns
+                    split_dfs = self._split_dataframe(df=df, split_threshold=self.column_split_threshold, axis="column")
+                else:
+                    # recursive split
+                    split_dfs = self._recursive_split(
+                        df=df,
+                        row_split_threshold=self.row_split_threshold,  # type: ignore
+                        column_split_threshold=self.column_split_threshold,  # type: ignore
+                    )
+
+            # check if no sub-tables were found
+            if len(split_dfs) == 0:
+                logger.warning(
+                    "No sub-tables found while splitting CSV Document with id {doc_id}. Skipping document.",
+                    doc_id=document.id,
+                )
+                continue
+
+            # Sort split_dfs first by row index, then by column index
+            split_dfs.sort(key=lambda dataframe: (dataframe.index[0], dataframe.columns[0]))
+
+            for split_id, split_df in enumerate(split_dfs):
+                split_documents.append(
+                    Document(
+                        content=split_df.to_csv(index=False, header=False, lineterminator="\n"),
+                        meta={
+                            **document.meta.copy(),
+                            "source_id": document.id,
+                            "row_idx_start": int(split_df.index[0]),
+                            "col_idx_start": int(split_df.columns[0]),
+                            "split_id": split_id,
+                        },
+                    )
+                )
+
+        return {"documents": split_documents}
+
+    @staticmethod
+    def _find_split_indices(
+        df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
+    ) -> list[tuple[int, int]]:
+        """
+        Finds the indices of consecutive empty rows or columns in a DataFrame.
+
+        :param df: DataFrame to split.
+        :param split_threshold: Minimum number of consecutive empty rows or columns to trigger a split.
+        :param axis: Axis along which to find empty elements. Either "row" or "column".
+        :return: List of indices where consecutive empty rows or columns start.
+        """
+        if axis == "row":
+            empty_elements = df[df.isnull().all(axis=1)].index.tolist()
+        else:
+            empty_elements = df.columns[df.isnull().all(axis=0)].tolist()
+
+        # If no empty elements found, return empty list
+        if len(empty_elements) == 0:
+            return []
+
+        # Identify groups of consecutive empty elements
+        split_indices = []
+        consecutive_count = 1
+        start_index = empty_elements[0]
+
+        for i in range(1, len(empty_elements)):
+            if empty_elements[i] == empty_elements[i - 1] + 1:
+                consecutive_count += 1
+            else:
+                if consecutive_count >= split_threshold:
+                    split_indices.append((start_index, empty_elements[i - 1]))
+                consecutive_count = 1
+                start_index = empty_elements[i]
+
+        # Handle the last group of consecutive elements
+        if consecutive_count >= split_threshold:
+            split_indices.append((start_index, empty_elements[-1]))
+
+        return split_indices
+
+    def _split_dataframe(
+        self, df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
+    ) -> list["pd.DataFrame"]:
+        """
+        Splits a DataFrame into sub-tables based on consecutive empty rows or columns exceeding `split_threshold`.
+
+        :param df: DataFrame to split.
+        :param split_threshold: Minimum number of consecutive empty rows or columns to trigger a split.
+        :param axis: Axis along which to split. Either "row" or "column".
+        :return: List of split DataFrames.
+        """
+        # Find indices of consecutive empty rows or columns
+        split_indices = self._find_split_indices(df=df, split_threshold=split_threshold, axis=axis)
+
+        # If no split_indices are found, return the original DataFrame
+        if len(split_indices) == 0:
+            return [df]
+
+        # Split the DataFrame at identified indices
+        sub_tables = []
+        table_start_idx = 0
+        df_length = df.shape[0] if axis == "row" else df.shape[1]
+        for empty_start_idx, empty_end_idx in split_indices + [(df_length, df_length)]:
+            # Avoid empty splits
+            if empty_start_idx - table_start_idx >= 1:
+                if axis == "row":
+                    sub_table = df.iloc[table_start_idx:empty_start_idx]
+                else:
+                    sub_table = df.iloc[:, table_start_idx:empty_start_idx]
+                if not sub_table.empty:
+                    sub_tables.append(sub_table)
+            table_start_idx = empty_end_idx + 1
+
+        return sub_tables
+
+    def _recursive_split(
+        self, df: "pd.DataFrame", row_split_threshold: int, column_split_threshold: int
+    ) -> list["pd.DataFrame"]:
+        """
+        Recursively splits a DataFrame.
+
+        Recursively splits a DataFrame first by empty rows, then by empty columns, and repeats the process
+        until no more splits are possible. Returns a list of DataFrames, each representing a fully separated sub-table.
+
+        :param df: A Pandas DataFrame representing a table (or multiple tables) extracted from a CSV.
+        :param row_split_threshold: The minimum number of consecutive empty rows required to trigger a split.
+        :param column_split_threshold: The minimum number of consecutive empty columns to trigger a split.
+        """
+
+        # Step 1: Split by rows
+        new_sub_tables = self._split_dataframe(df=df, split_threshold=row_split_threshold, axis="row")
+
+        # Step 2: Split by columns
+        final_tables = []
+        for table in new_sub_tables:
+            final_tables.extend(self._split_dataframe(df=table, split_threshold=column_split_threshold, axis="column"))
+
+        # Step 3: Recursively reapply splitting checked by whether any new empty rows appear after column split
+        result = []
+        for table in final_tables:
+            # Check if there are consecutive rows >= row_split_threshold now present
+            if len(self._find_split_indices(df=table, split_threshold=row_split_threshold, axis="row")) > 0:
+                result.extend(
+                    self._recursive_split(
+                        df=table, row_split_threshold=row_split_threshold, column_split_threshold=column_split_threshold
+                    )
+                )
+            else:
+                result.append(table)
+
+        return result
+
+    def _split_by_row(self, df: "pd.DataFrame") -> list["pd.DataFrame"]:
+        """Split each CSV row into a separate subtable"""
+        split_dfs = []
+        for idx, row in enumerate(df.itertuples(index=False)):
+            split_df = pd.DataFrame(row).T
+            split_df.index = [idx]  # Set the index of the new DataFrame to idx
+            split_dfs.append(split_df)
+        return split_dfs

haystack/components/preprocessors/document_cleaner.py

@@ -0,0 +1,352 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from collections.abc import Generator
+from copy import deepcopy
+from functools import partial, reduce
+from itertools import chain
+from typing import Literal
+from unicodedata import normalize
+
+from haystack import Document, component, logging
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class DocumentCleaner:
+    """
+    Cleans the text in the documents.
+
+    It removes extra whitespaces,
+    empty lines, specified substrings, regexes,
+    page headers and footers (in this order).
+
+    ### Usage example:
+
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import DocumentCleaner
+
+    doc = Document(content="This   is  a  document  to  clean\\n\\n\\nsubstring to remove")
+
+    cleaner = DocumentCleaner(remove_substrings = ["substring to remove"])
+    result = cleaner.run(documents=[doc])
+
+    assert result["documents"][0].content == "This is a document to clean "
+    ```
+    """
+
+    def __init__(
+        self,
+        remove_empty_lines: bool = True,
+        remove_extra_whitespaces: bool = True,
+        remove_repeated_substrings: bool = False,
+        keep_id: bool = False,
+        remove_substrings: list[str] | None = None,
+        remove_regex: str | None = None,
+        unicode_normalization: Literal["NFC", "NFKC", "NFD", "NFKD"] | None = None,
+        ascii_only: bool = False,
+        strip_whitespaces: bool = False,
+        replace_regexes: dict[str, str] | None = None,
+    ) -> None:
+        """
+        Initialize DocumentCleaner.
+
+        :param remove_empty_lines: If `True`, removes empty lines.
+        :param remove_extra_whitespaces: If `True`, removes extra whitespaces.
+        :param remove_repeated_substrings: If `True`, removes repeated substrings (headers and footers) from pages.
+            Pages must be separated by a form feed character "\\f",
+            which is supported by `TextFileToDocument` and `AzureOCRDocumentConverter`.
+        :param remove_substrings: List of substrings to remove from the text.
+        :param remove_regex: Regex to match and replace substrings by "".
+        :param keep_id: If `True`, keeps the IDs of the original documents.
+        :param unicode_normalization: Unicode normalization form to apply to the text.
+            Note: This will run before any other steps.
+        :param ascii_only: Whether to convert the text to ASCII only.
+            Will remove accents from characters and replace them with ASCII characters.
+            Other non-ASCII characters will be removed.
+            Note: This will run before any pattern matching or removal.
+        :param strip_whitespaces: If `True`, removes leading and trailing whitespace from the document content
+            using Python's `str.strip()`. Unlike `remove_extra_whitespaces`, this only affects the beginning
+            and end of the text, preserving internal whitespace (useful for markdown formatting).
+        :param replace_regexes: A dictionary mapping regex patterns to their replacement strings.
+            For example, `{r'\\n\\n+': '\\n'}` replaces multiple consecutive newlines with a single newline.
+            This is applied after `remove_regex` and allows custom replacements instead of just removal.
+        """
+
+        self._validate_params(unicode_normalization=unicode_normalization)
+
+        self.remove_empty_lines = remove_empty_lines
+        self.remove_extra_whitespaces = remove_extra_whitespaces
+        self.remove_repeated_substrings = remove_repeated_substrings
+        self.remove_substrings = remove_substrings
+        self.remove_regex = remove_regex
+        self.keep_id = keep_id
+        self.unicode_normalization = unicode_normalization
+        self.ascii_only = ascii_only
+        self.strip_whitespaces = strip_whitespaces
+        self.replace_regexes = replace_regexes
+
+    def _validate_params(self, unicode_normalization: str | None) -> None:
+        """
+        Validate the parameters of the DocumentCleaner.
+
+        :param unicode_normalization: Unicode normalization form to apply to the text.
+        :raises ValueError: if the parameters are not valid.
+        """
+        if unicode_normalization and unicode_normalization not in ["NFC", "NFKC", "NFD", "NFKD"]:
+            raise ValueError("unicode_normalization must be one of 'NFC', 'NFKC', 'NFD', 'NFKD'.")
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Cleans up the documents.
+
+        :param documents: List of Documents to clean.
+
+        :returns: A dictionary with the following key:
+            - `documents`: List of cleaned Documents.
+
+        :raises TypeError: if documents is not a list of Documents.
+        """
+        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
+            raise TypeError("DocumentCleaner expects a List of Documents as input.")
+
+        cleaned_docs = []
+        for doc in documents:
+            if doc.content is None:
+                logger.warning(
+                    "DocumentCleaner only cleans text documents but document.content for document ID"
+                    " {document_id} is None.",
+                    document_id=doc.id,
+                )
+                cleaned_docs.append(doc)
+                continue
+            text = doc.content
+
+            if self.unicode_normalization:
+                text = self._normalize_unicode(text, self.unicode_normalization)
+            if self.ascii_only:
+                text = self._ascii_only(text)
+            if self.remove_extra_whitespaces:
+                text = self._remove_extra_whitespaces(text)
+            if self.remove_empty_lines:
+                text = self._remove_empty_lines(text)
+            if self.remove_substrings:
+                text = self._remove_substrings(text, self.remove_substrings)
+            if self.remove_regex:
+                text = self._remove_regex(text, self.remove_regex)
+            if self.replace_regexes:
+                text = self._replace_regexes(text, self.replace_regexes)
+            if self.remove_repeated_substrings:
+                text = self._remove_repeated_substrings(text)
+            if self.strip_whitespaces:
+                text = text.strip()
+
+            clean_doc = Document(
+                id=doc.id if self.keep_id else "",
+                content=text,
+                blob=doc.blob,
+                meta=deepcopy(doc.meta),
+                score=doc.score,
+                embedding=doc.embedding,
+                sparse_embedding=doc.sparse_embedding,
+            )
+            cleaned_docs.append(clean_doc)
+
+        return {"documents": cleaned_docs}
+
+    def _normalize_unicode(self, text: str, form: Literal["NFC", "NFKC", "NFD", "NFKD"]) -> str:
+        """
+        Normalize the unicode of the text.
+
+        :param text: Text to normalize.
+        :param form: Unicode normalization form to apply to the text.
+            Options: "NFC", "NFKC", "NFD", "NFKD".
+        :returns: The normalized text.
+        """
+        return normalize(form, text)
+
+    def _ascii_only(self, text: str) -> str:
+        """
+        Convert the text to ASCII only.
+
+        Will remove accents from characters and replace them with ASCII characters.
+        Other non-ASCII characters will be removed.
+
+        :param text: Text to convert to ASCII only.
+        :returns: The text in ASCII only.
+        """
+
+        # First normalize the text to NFKD to separate the characters and their diacritics
+        # Then encode it to ASCII and ignore any characters that can't be encoded
+        return self._normalize_unicode(text, "NFKD").encode("ascii", "ignore").decode("utf-8")
+
+    def _remove_empty_lines(self, text: str) -> str:
+        """
+        Remove empty lines and lines that contain nothing but whitespaces from text.
+
+        :param text: Text to clean.
+        :returns: The text without empty lines.
+        """
+        pages = text.split("\f")
+        cleaned_pages = ["\n".join(line for line in page.split("\n") if line.strip()) for page in pages]
+        return "\f".join(cleaned_pages)
+
+    def _remove_extra_whitespaces(self, text: str) -> str:
+        """
+        Remove extra whitespaces from text.
+
+        :param text: Text to clean.
+        :returns: The text without extra whitespaces.
+        """
+        texts = text.split("\f")
+        cleaned_text = [re.sub(r"\s\s+", " ", text).strip() for text in texts]
+        return "\f".join(cleaned_text)
+
+    def _remove_regex(self, text: str, regex: str) -> str:
+        """
+        Remove substrings that match the specified regex from the text.
+
+        :param text: Text to clean.
+        :param regex: Regex to match and replace substrings by "".
+        :returns: The text without the substrings that match the regex.
+        """
+        texts = text.split("\f")
+        cleaned_text = [re.sub(regex, "", text).strip() for text in texts]
+        return "\f".join(cleaned_text)
+
+    def _replace_regexes(self, text: str, replace_regexes: dict[str, str]) -> str:
+        """
+        Replace substrings that match the specified regex patterns with custom replacement strings.
+
+        :param text: Text to clean.
+        :param replace_regexes: A dictionary mapping regex patterns to their replacement strings.
+        :returns: The text with the regex matches replaced by the specified strings.
+        """
+        pages = text.split("\f")
+        cleaned_pages = []
+        for page in pages:
+            for pattern, replacement in replace_regexes.items():
+                page = re.sub(pattern, replacement, page)
+            cleaned_pages.append(page)
+        return "\f".join(cleaned_pages)
+
+    def _remove_substrings(self, text: str, substrings: list[str]) -> str:
+        """
+        Remove all specified substrings from the text.
+
+        :param text: Text to clean.
+        :param substrings: Substrings to remove.
+        :returns: The text without the specified substrings.
+        """
+        for substring in substrings:
+            text = text.replace(substring, "")
+        return text
+
+    def _remove_repeated_substrings(self, text: str) -> str:
+        """
+        Remove any substrings from the text that occur repeatedly on every page. For example headers or footers.
+
+        Pages in the text need to be separated by form feed character "\f".
+        :param text: Text to clean.
+        :returns: The text without the repeated substrings.
+        """
+        return self._find_and_remove_header_footer(
+            text, n_chars=300, n_first_pages_to_ignore=1, n_last_pages_to_ignore=1
+        )
+
+    def _find_and_remove_header_footer(
+        self, text: str, n_chars: int, n_first_pages_to_ignore: int, n_last_pages_to_ignore: int
+    ) -> str:
+        """
+        Heuristic to find footers and headers across different pages by searching for the longest common string.
+
+        Pages in the text need to be separated by form feed character "\f".
+        For headers, we only search in the first n_chars characters (for footer: last n_chars).
+        Note: This heuristic uses exact matches and therefore works well for footers like "Copyright 2019 by XXX",
+         but won't detect "Page 3 of 4" or similar.
+
+        :param n_chars: The number of first/last characters where the header/footer shall be searched in.
+        :param n_first_pages_to_ignore: The number of first pages to ignore
+            (e.g. TOCs often don't contain footer/header).
+        :param n_last_pages_to_ignore: The number of last pages to ignore.
+        :returns: The text without the found headers and footers.
+        """
+
+        pages = text.split("\f")
+
+        # header
+        start_of_pages = [p[:n_chars] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]
+        found_header = self._find_longest_common_ngram(start_of_pages)
+        if found_header:
+            pages = [page.replace(found_header, "") for page in pages]
+
+        # footer
+        end_of_pages = [p[-n_chars:] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]
+        found_footer = self._find_longest_common_ngram(end_of_pages)
+        if found_footer:
+            pages = [page.replace(found_footer, "") for page in pages]
+
+        logger.debug(
+            "Removed header '{header}' and footer '{footer}' in document", header=found_header, footer=found_footer
+        )
+        return "\f".join(pages)
+
+    def _ngram(self, seq: str, n: int) -> Generator[str, None, None]:
+        """
+        Return all ngrams of length n from a text sequence. Each ngram consists of n words split by whitespace.
+
+        :param seq: The sequence to generate ngrams from.
+        :param n: The length of the ngrams to generate.
+        :returns: A Generator generating all ngrams of length n from the given sequence.
+        """
+
+        # In order to maintain the original whitespace, but still consider \n and \t for n-gram tokenization,
+        # we add a space here and remove it after creation of the ngrams again (see below)
+        seq = seq.replace("\n", " \n")
+        seq = seq.replace("\t", " \t")
+
+        words = seq.split(" ")
+        return (" ".join(words[i : i + n]).replace(" \n", "\n").replace(" \t", "\t") for i in range(len(words) - n + 1))
+
+    def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> set[str]:
+        """
+        Generates all possible ngrams from a given sequence of text.
+
+        Considering all ngram lengths between the minimum and maximum length.
+
+        :param seq: The sequence to generate ngrams from.
+        :param min_ngram: The minimum length of ngram to consider.
+        :param max_ngram: The maximum length of ngram to consider.
+        :returns: A set of all ngrams from the given sequence.
+        """
+        lengths = range(min_ngram, max_ngram) if max_ngram else range(min_ngram, len(seq))
+        ngrams = map(partial(self._ngram, seq), lengths)
+        return set(chain.from_iterable(ngrams))
+
+    def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, max_ngram: int = 30) -> str:
+        """
+        Find the longest common ngram across a list of text sequences (e.g. start of pages).
+
+        Considering all ngram lengths between the minimum and maximum length. Helpful for finding footers, headers etc.
+        Empty sequences are ignored.
+
+        :param sequences: The list of strings that shall be searched for common n_grams.
+        :param max_ngram: The maximum length of ngram to consider.
+        :param min_ngram: The minimum length of ngram to consider.
+        :returns: The longest ngram that all sequences have in common.
+        """
+        sequences = [s for s in sequences if s]  # filter empty sequences
+        if len(sequences) < 2:
+            # a single sequence has no ngram "in common" with any other; treating
+            # its own longest ngram as a repeated header/footer would wipe it
+            return ""
+        seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)
+        intersection = reduce(set.intersection, seqs_ngrams)
+
+        longest = max(intersection, key=len, default="")
+        return longest if longest.strip() else ""

haystack/components/preprocessors/document_preprocessor.py

@@ -0,0 +1,198 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Literal
+
+from haystack import Document, Pipeline, default_from_dict, default_to_dict, super_component
+from haystack.components.preprocessors.document_cleaner import DocumentCleaner
+from haystack.components.preprocessors.document_splitter import DocumentSplitter, Language
+from haystack.utils import deserialize_callable, serialize_callable
+
+
+@super_component
+class DocumentPreprocessor:
+    """
+    A SuperComponent that first splits and then cleans documents.
+
+    This component consists of a DocumentSplitter followed by a DocumentCleaner in a single pipeline.
+    It takes a list of documents as input and returns a processed list of documents.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import DocumentPreprocessor
+
+    doc = Document(content="I love pizza!")
+    preprocessor = DocumentPreprocessor()
+    result = preprocessor.run(documents=[doc])
+    print(result["documents"])
+    ```
+    """
+
+    def __init__(  # noqa: PLR0913 (too-many-arguments)
+        self,
+        *,
+        # --- DocumentSplitter arguments ---
+        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence"] = "word",
+        split_length: int = 250,
+        split_overlap: int = 0,
+        split_threshold: int = 0,
+        splitting_function: Callable[[str], list[str]] | None = None,
+        respect_sentence_boundary: bool = False,
+        language: Language = "en",
+        use_split_rules: bool = True,
+        extend_abbreviations: bool = True,
+        # --- DocumentCleaner arguments ---
+        remove_empty_lines: bool = True,
+        remove_extra_whitespaces: bool = True,
+        remove_repeated_substrings: bool = False,
+        keep_id: bool = False,
+        remove_substrings: list[str] | None = None,
+        remove_regex: str | None = None,
+        unicode_normalization: Literal["NFC", "NFKC", "NFD", "NFKD"] | None = None,
+        ascii_only: bool = False,
+    ) -> None:
+        """
+        Initialize a DocumentPreProcessor that first splits and then cleans documents.
+
+        **Splitter Parameters**:
+        :param split_by: The unit of splitting: "function", "page", "passage", "period", "word", "line", or "sentence".
+        :param split_length: The maximum number of units (words, lines, pages, and so on) in each split.
+        :param split_overlap: The number of overlapping units between consecutive splits.
+        :param split_threshold: The minimum number of units per split. If a split is smaller than this, it's merged
+            with the previous split.
+        :param splitting_function: A custom function for splitting if `split_by="function"`.
+        :param respect_sentence_boundary: If `True`, splits by words but tries not to break inside a sentence.
+        :param language: Language used by the sentence tokenizer if `split_by="sentence"` or
+            `respect_sentence_boundary=True`.
+        :param use_split_rules: Whether to apply additional splitting heuristics for the sentence splitter.
+        :param extend_abbreviations: Whether to extend the sentence splitter with curated abbreviations for certain
+            languages.
+
+        **Cleaner Parameters**:
+        :param remove_empty_lines: If `True`, removes empty lines.
+        :param remove_extra_whitespaces: If `True`, removes extra whitespaces.
+        :param remove_repeated_substrings: If `True`, removes repeated substrings like headers/footers across pages.
+        :param keep_id: If `True`, keeps the original document IDs.
+        :param remove_substrings: A list of strings to remove from the document content.
+        :param remove_regex: A regex pattern whose matches will be removed from the document content.
+        :param unicode_normalization: Unicode normalization form to apply to the text, for example `"NFC"`.
+        :param ascii_only: If `True`, converts text to ASCII only.
+        """
+        # Store arguments for serialization
+        self.remove_empty_lines = remove_empty_lines
+        self.remove_extra_whitespaces = remove_extra_whitespaces
+        self.remove_repeated_substrings = remove_repeated_substrings
+        self.keep_id = keep_id
+        self.remove_substrings = remove_substrings
+        self.remove_regex = remove_regex
+        self.unicode_normalization = unicode_normalization
+        self.ascii_only = ascii_only
+
+        self.split_by = split_by
+        self.split_length = split_length
+        self.split_overlap = split_overlap
+        self.split_threshold = split_threshold
+        self.splitting_function = splitting_function
+        self.respect_sentence_boundary = respect_sentence_boundary
+        self.language = language
+        self.use_split_rules = use_split_rules
+        self.extend_abbreviations = extend_abbreviations
+
+        # Instantiate sub-components
+        splitter = DocumentSplitter(
+            split_by=self.split_by,
+            split_length=self.split_length,
+            split_overlap=self.split_overlap,
+            split_threshold=self.split_threshold,
+            splitting_function=self.splitting_function,
+            respect_sentence_boundary=self.respect_sentence_boundary,
+            language=self.language,
+            use_split_rules=self.use_split_rules,
+            extend_abbreviations=self.extend_abbreviations,
+        )
+
+        cleaner = DocumentCleaner(
+            remove_empty_lines=self.remove_empty_lines,
+            remove_extra_whitespaces=self.remove_extra_whitespaces,
+            remove_repeated_substrings=self.remove_repeated_substrings,
+            keep_id=self.keep_id,
+            remove_substrings=self.remove_substrings,
+            remove_regex=self.remove_regex,
+            unicode_normalization=self.unicode_normalization,
+            ascii_only=self.ascii_only,
+        )
+
+        # Build the Pipeline
+        pp = Pipeline()
+
+        pp.add_component("splitter", splitter)
+        pp.add_component("cleaner", cleaner)
+
+        # Connect the splitter output to cleaner
+        pp.connect("splitter.documents", "cleaner.documents")
+        self.pipeline = pp
+
+        # Define how pipeline inputs/outputs map to sub-component inputs/outputs
+        self.input_mapping = {
+            # The pipeline input "documents" feeds into "splitter.documents"
+            "documents": ["splitter.documents"]
+        }
+        # The pipeline output "documents" comes from "cleaner.documents"
+        self.output_mapping = {"cleaner.documents": "documents"}
+
+    if TYPE_CHECKING:
+        # fake method, never executed, but static analyzers will not complain about missing method
+        def run(self, *, documents: list[Document]) -> dict[str, list[Document]]:  # noqa: D102
+            ...
+        def warm_up(self) -> None:  # noqa: D102
+            ...
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize SuperComponent to a dictionary.
+
+        :return:
+            Dictionary with serialized data.
+        """
+        splitting_function = None
+        if self.splitting_function is not None:
+            splitting_function = serialize_callable(self.splitting_function)
+
+        return default_to_dict(
+            self,
+            remove_empty_lines=self.remove_empty_lines,
+            remove_extra_whitespaces=self.remove_extra_whitespaces,
+            remove_repeated_substrings=self.remove_repeated_substrings,
+            keep_id=self.keep_id,
+            remove_substrings=self.remove_substrings,
+            remove_regex=self.remove_regex,
+            unicode_normalization=self.unicode_normalization,
+            ascii_only=self.ascii_only,
+            split_by=self.split_by,
+            split_length=self.split_length,
+            split_overlap=self.split_overlap,
+            split_threshold=self.split_threshold,
+            splitting_function=splitting_function,
+            respect_sentence_boundary=self.respect_sentence_boundary,
+            language=self.language,
+            use_split_rules=self.use_split_rules,
+            extend_abbreviations=self.extend_abbreviations,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "DocumentPreprocessor":
+        """
+        Deserializes the SuperComponent from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized SuperComponent.
+        """
+        splitting_function = data["init_parameters"].get("splitting_function", None)
+        if splitting_function:
+            data["init_parameters"]["splitting_function"] = deserialize_callable(splitting_function)
+        return default_from_dict(cls, data)

haystack/components/preprocessors/document_splitter.py

@@ -0,0 +1,499 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from copy import deepcopy
+from typing import Any, Literal
+
+from more_itertools import windowed
+
+from haystack import Document, component, logging
+from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter, nltk_imports
+from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.utils import deserialize_callable, serialize_callable
+
+logger = logging.getLogger(__name__)
+
+# mapping of split by character, 'function' and 'sentence' don't split by character
+_CHARACTER_SPLIT_BY_MAPPING = {"page": "\f", "passage": "\n\n", "period": ".", "word": " ", "line": "\n"}
+
+
+@component
+class DocumentSplitter:
+    """
+    Splits long documents into smaller chunks.
+
+    This is a common preprocessing step during indexing. It helps Embedders create meaningful semantic representations
+    and prevents exceeding language model context limits.
+
+    The DocumentSplitter is compatible with the following DocumentStores:
+    - [Astra](https://docs.haystack.deepset.ai/docs/astradocumentstore)
+    - [Chroma](https://docs.haystack.deepset.ai/docs/chromadocumentstore) limited support, overlapping information is
+      not stored
+    - [Elasticsearch](https://docs.haystack.deepset.ai/docs/elasticsearch-document-store)
+    - [OpenSearch](https://docs.haystack.deepset.ai/docs/opensearch-document-store)
+    - [Pgvector](https://docs.haystack.deepset.ai/docs/pgvectordocumentstore)
+    - [Pinecone](https://docs.haystack.deepset.ai/docs/pinecone-document-store) limited support, overlapping
+       information is not stored
+    - [Qdrant](https://docs.haystack.deepset.ai/docs/qdrant-document-store)
+    - [Weaviate](https://docs.haystack.deepset.ai/docs/weaviatedocumentstore)
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import DocumentSplitter
+
+    doc = Document(content="Moonlight shimmered softly, wolves howled nearby, night enveloped everything.")
+
+    splitter = DocumentSplitter(split_by="word", split_length=3, split_overlap=0)
+    result = splitter.run(documents=[doc])
+    ```
+    """
+
+    def __init__(
+        self,
+        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence"] = "word",
+        split_length: int = 200,
+        split_overlap: int = 0,
+        split_threshold: int = 0,
+        splitting_function: Callable[[str], list[str]] | None = None,
+        respect_sentence_boundary: bool = False,
+        language: Language = "en",
+        use_split_rules: bool = True,
+        extend_abbreviations: bool = True,
+        *,
+        skip_empty_documents: bool = True,
+    ) -> None:
+        """
+        Initialize DocumentSplitter.
+
+        :param split_by: The unit for splitting your documents. Choose from:
+            - `word` for splitting by spaces (" ")
+            - `period` for splitting by periods (".")
+            - `page` for splitting by form feed ("\\f")
+            - `passage` for splitting by double line breaks ("\\n\\n")
+            - `line` for splitting each line ("\\n")
+            - `sentence` for splitting by NLTK sentence tokenizer
+
+        :param split_length: The maximum number of units in each split.
+        :param split_overlap: The number of overlapping units for each split.
+        :param split_threshold: The minimum number of units per split. If a split has fewer units
+            than the threshold, it's attached to the previous split.
+        :param splitting_function: Necessary when `split_by` is set to "function".
+            This is a function which must accept a single `str` as input and return a `list` of `str` as output,
+            representing the chunks after splitting.
+        :param respect_sentence_boundary: Choose whether to respect sentence boundaries when splitting by "word".
+            If True, uses NLTK to detect sentence boundaries, ensuring splits occur only between sentences.
+        :param language: Choose the language for the NLTK tokenizer. The default is English ("en").
+        :param use_split_rules: Choose whether to use additional split rules when splitting by `sentence`.
+        :param extend_abbreviations: Choose whether to extend NLTK's PunktTokenizer abbreviations with a list
+            of curated abbreviations, if available. This is currently supported for English ("en") and German ("de").
+        :param skip_empty_documents: Choose whether to skip documents with empty content. Default is True.
+            Set to False when downstream components in the Pipeline (like LLMDocumentContentExtractor) can extract text
+            from non-textual documents.
+        """
+
+        self.split_by = split_by
+        self.split_length = split_length
+        self.split_overlap = split_overlap
+        self.split_threshold = split_threshold
+        self.splitting_function = splitting_function
+        self.respect_sentence_boundary = respect_sentence_boundary
+        self.language = language
+        self.use_split_rules = use_split_rules
+        self.extend_abbreviations = extend_abbreviations
+        self.skip_empty_documents = skip_empty_documents
+
+        self._init_checks(
+            split_by=split_by,
+            split_length=split_length,
+            split_overlap=split_overlap,
+            splitting_function=splitting_function,
+            respect_sentence_boundary=respect_sentence_boundary,
+        )
+        self._use_sentence_splitter = split_by == "sentence" or (respect_sentence_boundary and split_by == "word")
+        if self._use_sentence_splitter:
+            nltk_imports.check()
+            self.sentence_splitter: SentenceSplitter | None = None
+
+    def _init_checks(
+        self,
+        *,
+        split_by: str,
+        split_length: int,
+        split_overlap: int,
+        splitting_function: Callable | None,
+        respect_sentence_boundary: bool,
+    ) -> None:
+        """
+        Validates initialization parameters for DocumentSplitter.
+
+        :param split_by: The unit for splitting documents
+        :param split_length: The maximum number of units in each split
+        :param split_overlap: The number of overlapping units for each split
+        :param splitting_function: Custom function for splitting when split_by="function"
+        :param respect_sentence_boundary: Whether to respect sentence boundaries when splitting
+        :raises ValueError: If any parameter is invalid
+        """
+        valid_split_by = ["function", "page", "passage", "period", "word", "line", "sentence"]
+        if split_by not in valid_split_by:
+            raise ValueError(f"split_by must be one of {', '.join(valid_split_by)}.")
+
+        if split_by == "function" and splitting_function is None:
+            raise ValueError("When 'split_by' is set to 'function', a valid 'splitting_function' must be provided.")
+
+        if split_length <= 0:
+            raise ValueError("split_length must be greater than 0.")
+
+        if split_overlap < 0:
+            raise ValueError("split_overlap must be greater than or equal to 0.")
+
+        if split_overlap >= split_length:
+            raise ValueError("split_overlap must be less than split_length.")
+
+        if respect_sentence_boundary and split_by != "word":
+            logger.warning(
+                "The 'respect_sentence_boundary' option is only supported for `split_by='word'`. "
+                "The option `respect_sentence_boundary` will be set to `False`."
+            )
+            self.respect_sentence_boundary = False
+
+    def warm_up(self) -> None:
+        """
+        Warm up the DocumentSplitter by loading the sentence tokenizer.
+        """
+        if self._use_sentence_splitter and self.sentence_splitter is None:
+            self.sentence_splitter = SentenceSplitter(
+                language=self.language,
+                use_split_rules=self.use_split_rules,
+                extend_abbreviations=self.extend_abbreviations,
+                keep_white_spaces=True,
+            )
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Split documents into smaller parts.
+
+        Splits documents by the unit expressed in `split_by`, with a length of `split_length`
+        and an overlap of `split_overlap`.
+
+        :param documents: The documents to split.
+        :returns: A dictionary with the following key:
+            - `documents`: List of documents with the split texts. Each document includes:
+                - A metadata field `source_id` to track the original document.
+                - A metadata field `page_number` to track the original page number.
+                - All other metadata copied from the original document.
+
+        :raises TypeError: if the input is not a list of Documents.
+        :raises ValueError: if the content of a document is None.
+        """
+        if self._use_sentence_splitter and self.sentence_splitter is None:
+            self.warm_up()
+
+        if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
+            raise TypeError("DocumentSplitter expects a List of Documents as input.")
+
+        split_docs: list[Document] = []
+        for doc in documents:
+            if doc.content is None:
+                raise ValueError(
+                    f"DocumentSplitter only works with text documents but content for document ID {doc.id} is None."
+                )
+            if doc.content == "" and self.skip_empty_documents:
+                logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
+                continue
+
+            split_docs += self._split_document(doc)
+        return {"documents": split_docs}
+
+    def _split_document(self, doc: Document) -> list[Document]:
+        if self.split_by == "sentence" or self.respect_sentence_boundary:
+            return self._split_by_nltk_sentence(doc)
+
+        if self.split_by == "function" and self.splitting_function is not None:
+            return self._split_by_function(doc)
+
+        return self._split_by_character(doc)
+
+    def _split_by_nltk_sentence(self, doc: Document) -> list[Document]:
+        split_docs = []
+
+        result = self.sentence_splitter.split_sentences(doc.content)  # type: ignore # None check is done in run()
+        units = [sentence["sentence"] for sentence in result]
+
+        if self.respect_sentence_boundary:
+            text_splits, splits_pages, splits_start_idxs = self._concatenate_sentences_based_on_word_amount(
+                sentences=units, split_length=self.split_length, split_overlap=self.split_overlap
+            )
+        else:
+            text_splits, splits_pages, splits_start_idxs = self._concatenate_units(
+                elements=units,
+                split_length=self.split_length,
+                split_overlap=self.split_overlap,
+                split_threshold=self.split_threshold,
+            )
+        metadata = deepcopy(doc.meta)
+        metadata["source_id"] = doc.id
+        split_docs += self._create_docs_from_splits(
+            text_splits=text_splits, splits_pages=splits_pages, splits_start_idxs=splits_start_idxs, meta=metadata
+        )
+
+        return split_docs
+
+    def _split_by_character(self, doc: Document) -> list[Document]:
+        split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
+        units = doc.content.split(split_at)  # type: ignore[union-attr]
+        # Add the delimiter back to all units except the last one
+        for i in range(len(units) - 1):
+            units[i] += split_at
+        text_splits, splits_pages, splits_start_idxs = self._concatenate_units(
+            units, self.split_length, self.split_overlap, self.split_threshold
+        )
+        metadata = deepcopy(doc.meta)
+        metadata["source_id"] = doc.id
+        return self._create_docs_from_splits(
+            text_splits=text_splits, splits_pages=splits_pages, splits_start_idxs=splits_start_idxs, meta=metadata
+        )
+
+    def _split_by_function(self, doc: Document) -> list[Document]:
+        # the check for None is done already in the run method
+        splits = self.splitting_function(doc.content)  # type: ignore
+        docs: list[Document] = []
+        for s in splits:
+            meta = deepcopy(doc.meta)
+            meta["source_id"] = doc.id
+            docs.append(Document(content=s, meta=meta))
+        return docs
+
+    def _concatenate_units(
+        self, elements: list[str], split_length: int, split_overlap: int, split_threshold: int
+    ) -> tuple[list[str], list[int], list[int]]:
+        """
+        Concatenates the elements into parts of split_length units.
+
+        Keeps track of the original page number that each element belongs. If the length of the current units is less
+        than the pre-defined `split_threshold`, it does not create a new split. Instead, it concatenates the current
+        units with the last split, preventing the creation of excessively small splits.
+        """
+
+        text_splits: list[str] = []
+        splits_pages: list[int] = []
+        splits_start_idxs: list[int] = []
+        cur_start_idx = 0
+        cur_page = 1
+        segments = windowed(elements, n=split_length, step=split_length - split_overlap)
+
+        for seg in segments:
+            current_units = [unit for unit in seg if unit is not None]
+            txt = "".join(current_units)
+
+            # check if length of current units is below split_threshold
+            if len(current_units) < split_threshold and len(text_splits) > 0:
+                # concatenate the last split with the current one
+                text_splits[-1] += txt
+
+            # NOTE: If skip_empty_documents is True, this line skips documents that have content=""
+            elif not self.skip_empty_documents or len(txt) > 0:
+                text_splits.append(txt)
+                splits_pages.append(cur_page)
+                splits_start_idxs.append(cur_start_idx)
+
+            processed_units = current_units[: split_length - split_overlap]
+            cur_start_idx += len("".join(processed_units))
+
+            if self.split_by == "page":
+                num_page_breaks = len(processed_units)
+            else:
+                num_page_breaks = sum(processed_unit.count("\f") for processed_unit in processed_units)
+
+            cur_page += num_page_breaks
+
+        return text_splits, splits_pages, splits_start_idxs
+
+    def _create_docs_from_splits(
+        self, text_splits: list[str], splits_pages: list[int], splits_start_idxs: list[int], meta: dict[str, Any]
+    ) -> list[Document]:
+        """
+        Creates Document objects from splits enriching them with page number and the metadata of the original document.
+        """
+        documents: list[Document] = []
+
+        for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs, strict=True)):
+            copied_meta = deepcopy(meta)
+            copied_meta["page_number"] = splits_pages[i]
+            copied_meta["split_id"] = i
+            copied_meta["split_idx_start"] = split_idx
+            doc = Document(content=txt, meta=copied_meta)
+            documents.append(doc)
+
+            if self.split_overlap <= 0:
+                continue
+
+            doc.meta["_split_overlap"] = []
+
+            if i == 0:
+                continue
+
+            doc_start_idx = splits_start_idxs[i]
+            previous_doc = documents[i - 1]
+            previous_doc_start_idx = splits_start_idxs[i - 1]
+            self._add_split_overlap_information(doc, doc_start_idx, previous_doc, previous_doc_start_idx)
+
+        return documents
+
+    @staticmethod
+    def _add_split_overlap_information(
+        current_doc: Document, current_doc_start_idx: int, previous_doc: Document, previous_doc_start_idx: int
+    ) -> None:
+        """
+        Adds split overlap information to the current and previous Document's meta.
+
+        :param current_doc: The Document that is being split.
+        :param current_doc_start_idx: The starting index of the current Document.
+        :param previous_doc: The Document that was split before the current Document.
+        :param previous_doc_start_idx: The starting index of the previous Document.
+        """
+        overlapping_range = (current_doc_start_idx - previous_doc_start_idx, len(previous_doc.content))  # type: ignore
+
+        if overlapping_range[0] < overlapping_range[1]:
+            overlapping_str = previous_doc.content[overlapping_range[0] : overlapping_range[1]]  # type: ignore
+
+            if current_doc.content.startswith(overlapping_str):  # type: ignore
+                # add split overlap information to this Document regarding the previous Document
+                current_doc.meta["_split_overlap"].append({"doc_id": previous_doc.id, "range": overlapping_range})
+
+                # add split overlap information to previous Document regarding this Document
+                overlapping_range = (0, overlapping_range[1] - overlapping_range[0])
+                previous_doc.meta["_split_overlap"].append({"doc_id": current_doc.id, "range": overlapping_range})
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+        """
+        serialized = default_to_dict(
+            self,
+            split_by=self.split_by,
+            split_length=self.split_length,
+            split_overlap=self.split_overlap,
+            split_threshold=self.split_threshold,
+            respect_sentence_boundary=self.respect_sentence_boundary,
+            language=self.language,
+            use_split_rules=self.use_split_rules,
+            extend_abbreviations=self.extend_abbreviations,
+            skip_empty_documents=self.skip_empty_documents,
+        )
+        if self.splitting_function:
+            serialized["init_parameters"]["splitting_function"] = serialize_callable(self.splitting_function)
+        return serialized
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "DocumentSplitter":
+        """
+        Deserializes the component from a dictionary.
+        """
+        init_params = data.get("init_parameters", {})
+
+        splitting_function = init_params.get("splitting_function", None)
+        if splitting_function:
+            init_params["splitting_function"] = deserialize_callable(splitting_function)
+
+        return default_from_dict(cls, data)
+
+    @staticmethod
+    def _concatenate_sentences_based_on_word_amount(
+        sentences: list[str], split_length: int, split_overlap: int
+    ) -> tuple[list[str], list[int], list[int]]:
+        """
+        Groups the sentences into chunks of `split_length` words while respecting sentence boundaries.
+
+        This function is only used when splitting by `word` and `respect_sentence_boundary` is set to `True`, i.e.:
+        with NLTK sentence tokenizer.
+
+        :param sentences: The list of sentences to split.
+        :param split_length: The maximum number of words in each split.
+        :param split_overlap: The number of overlapping words in each split.
+        :returns: A tuple containing the concatenated sentences, the start page numbers, and the start indices.
+        """
+        # chunk information
+        chunk_word_count = 0
+        chunk_starting_page_number = 1
+        chunk_start_idx = 0
+        current_chunk: list[str] = []
+        # output lists
+        split_start_page_numbers = []
+        list_of_splits: list[list[str]] = []
+        split_start_indices = []
+
+        for sentence_idx, sentence in enumerate(sentences):
+            current_chunk.append(sentence)
+            chunk_word_count += len(sentence.split())
+            next_sentence_word_count = (
+                len(sentences[sentence_idx + 1].split()) if sentence_idx < len(sentences) - 1 else 0
+            )
+
+            # Number of words in the current chunk plus the next sentence is larger than the split_length,
+            # or we reached the last sentence
+            if (chunk_word_count + next_sentence_word_count) > split_length or sentence_idx == len(sentences) - 1:
+                #  Save current chunk and start a new one
+                list_of_splits.append(current_chunk)
+                split_start_page_numbers.append(chunk_starting_page_number)
+                split_start_indices.append(chunk_start_idx)
+
+                # Get the number of sentences that overlap with the next chunk
+                num_sentences_to_keep = DocumentSplitter._number_of_sentences_to_keep(
+                    sentences=current_chunk, split_length=split_length, split_overlap=split_overlap
+                )
+                # Set up information for the new chunk
+                if num_sentences_to_keep > 0:
+                    # Processed sentences are the ones that are not overlapping with the next chunk
+                    processed_sentences = current_chunk[:-num_sentences_to_keep]
+                    chunk_starting_page_number += sum(sent.count("\f") for sent in processed_sentences)
+                    chunk_start_idx += len("".join(processed_sentences))
+                    # Next chunk starts with the sentences that were overlapping with the previous chunk
+                    current_chunk = current_chunk[-num_sentences_to_keep:]
+                    chunk_word_count = sum(len(s.split()) for s in current_chunk)
+                else:
+                    # Here processed_sentences is the same as current_chunk since there is no overlap
+                    chunk_starting_page_number += sum(sent.count("\f") for sent in current_chunk)
+                    chunk_start_idx += len("".join(current_chunk))
+                    current_chunk = []
+                    chunk_word_count = 0
+
+        # Concatenate the sentences together within each split
+        text_splits = []
+        for split in list_of_splits:
+            text = "".join(split)
+            if len(text) > 0:
+                text_splits.append(text)
+
+        return text_splits, split_start_page_numbers, split_start_indices
+
+    @staticmethod
+    def _number_of_sentences_to_keep(sentences: list[str], split_length: int, split_overlap: int) -> int:
+        """
+        Returns the number of sentences to keep in the next chunk based on the `split_overlap` and `split_length`.
+
+        :param sentences: The list of sentences to split.
+        :param split_length: The maximum number of words in each split.
+        :param split_overlap: The number of overlapping words in each split.
+        :returns: The number of sentences to keep in the next chunk.
+        """
+        # If the split_overlap is 0, we don't need to keep any sentences
+        if split_overlap == 0:
+            return 0
+
+        num_sentences_to_keep = 0
+        num_words = 0
+        # Next overlapping Document should not start exactly the same as the previous one, so we skip the first sentence
+        for sent in reversed(sentences[1:]):
+            num_words += len(sent.split())
+            # If the number of words is larger than the split_length then don't add any more sentences
+            if num_words > split_length:
+                break
+            num_sentences_to_keep += 1
+            if num_words > split_overlap:
+                break
+        return num_sentences_to_keep

haystack/components/preprocessors/embedding_based_document_splitter.py

@@ -0,0 +1,545 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from asyncio import gather
+from collections.abc import Awaitable
+from copy import deepcopy
+from itertools import chain
+from typing import Any
+
+import numpy as np
+
+from haystack import Document, component, logging
+from haystack.components.embedders.types import DocumentEmbedder
+from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter
+from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.deserialization import deserialize_component_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class EmbeddingBasedDocumentSplitter:
+    """
+    Splits documents based on embedding similarity using cosine distances between sequential sentence groups.
+
+    This component first splits text into sentences, optionally groups them, calculates embeddings for each group,
+    and then uses cosine distance between sequential embeddings to determine split points. Any distance above
+    the specified percentile is treated as a break point. The component also tracks page numbers based on form feed
+    characters (`\f`) in the original document.
+
+    This component is inspired by [5 Levels of Text Splitting](
+        https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb
+    ) by Greg Kamradt.
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.embedders import OpenAIDocumentEmbedder
+    from haystack.components.preprocessors import EmbeddingBasedDocumentSplitter
+
+    # Create a document with content that has a clear topic shift
+    doc = Document(
+        content="This is a first sentence. This is a second sentence. This is a third sentence. "
+        "Completely different topic. The same completely different topic."
+    )
+
+    # Initialize the embedder to calculate semantic similarities
+    embedder = OpenAIDocumentEmbedder()
+
+    # Configure the splitter with parameters that control splitting behavior
+    splitter = EmbeddingBasedDocumentSplitter(
+        document_embedder=embedder,
+        sentences_per_group=2,      # Group 2 sentences before calculating embeddings
+        percentile=0.95,            # Split when cosine distance exceeds 95th percentile
+        min_length=50,              # Merge splits shorter than 50 characters
+        max_length=1000             # Further split chunks longer than 1000 characters
+    )
+    result = splitter.run(documents=[doc])
+
+    # The result contains a list of Document objects, each representing a semantic chunk
+    # Each split document includes metadata: source_id, split_id, and page_number
+    print(f"Original document split into {len(result['documents'])} chunks")
+    for i, split_doc in enumerate(result['documents']):
+        print(f"Chunk {i}: {split_doc.content[:50]}...")
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        document_embedder: DocumentEmbedder,
+        sentences_per_group: int = 3,
+        percentile: float = 0.95,
+        min_length: int = 50,
+        max_length: int = 1000,
+        language: Language = "en",
+        use_split_rules: bool = True,
+        extend_abbreviations: bool = True,
+    ) -> None:
+        """
+        Initialize EmbeddingBasedDocumentSplitter.
+
+        :param document_embedder: The DocumentEmbedder to use for calculating embeddings.
+        :param sentences_per_group: Number of sentences to group together before embedding.
+        :param percentile: Percentile threshold for cosine distance. Distances above this percentile
+            are treated as break points.
+        :param min_length: Minimum length of splits in characters. Splits below this length will be merged.
+        :param max_length: Maximum length of splits in characters. Splits above this length will be recursively split.
+        :param language: Language for sentence tokenization.
+        :param use_split_rules: Whether to use additional split rules for sentence tokenization. Applies additional
+            split rules from SentenceSplitter to the sentence spans.
+        :param extend_abbreviations: If True, the abbreviations used by NLTK's PunktTokenizer are extended by a list
+            of curated abbreviations. Currently supported languages are: en, de.
+            If False, the default abbreviations are used.
+        """
+        self.document_embedder = document_embedder
+
+        if sentences_per_group <= 0:
+            raise ValueError("sentences_per_group must be greater than 0.")
+        self.sentences_per_group = sentences_per_group
+
+        if not 0.0 <= percentile <= 1.0:
+            raise ValueError("percentile must be between 0.0 and 1.0.")
+        self.percentile = percentile
+
+        if min_length < 0:
+            raise ValueError("min_length must be greater than or equal to 0.")
+        self.min_length = min_length
+
+        if max_length <= min_length:
+            raise ValueError("max_length must be greater than min_length.")
+        self.max_length = max_length
+
+        self.language = language
+        self.use_split_rules = use_split_rules
+        self.extend_abbreviations = extend_abbreviations
+        self.sentence_splitter: SentenceSplitter | None = None
+
+    def warm_up(self) -> None:
+        """
+        Warm up the component by initializing the sentence splitter and the document embedder.
+        """
+        if self.sentence_splitter is None:
+            self.sentence_splitter = SentenceSplitter(
+                language=self.language,
+                use_split_rules=self.use_split_rules,
+                extend_abbreviations=self.extend_abbreviations,
+                keep_white_spaces=True,
+            )
+        if hasattr(self.document_embedder, "warm_up"):
+            self.document_embedder.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the component on the serving event loop.
+
+        Initializes the sentence splitter and warms up the document embedder using its async warm-up path when
+        available, falling back to the synchronous one otherwise.
+        """
+        if self.sentence_splitter is None:
+            self.sentence_splitter = SentenceSplitter(
+                language=self.language,
+                use_split_rules=self.use_split_rules,
+                extend_abbreviations=self.extend_abbreviations,
+                keep_white_spaces=True,
+            )
+        if hasattr(self.document_embedder, "warm_up_async"):
+            await self.document_embedder.warm_up_async()
+        elif hasattr(self.document_embedder, "warm_up"):
+            self.document_embedder.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the document embedder's resources.
+        """
+        if hasattr(self.document_embedder, "close"):
+            self.document_embedder.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the document embedder's async resources.
+        """
+        if hasattr(self.document_embedder, "close_async"):
+            await self.document_embedder.close_async()
+        elif hasattr(self.document_embedder, "close"):
+            self.document_embedder.close()
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Split documents based on embedding similarity.
+
+        :param documents: The documents to split.
+        :returns: A dictionary with the following key:
+            - `documents`: List of documents with the split texts. Each document includes:
+                - A metadata field `source_id` to track the original document.
+                - A metadata field `split_id` to track the split number.
+                - A metadata field `page_number` to track the original page number.
+                - All other metadata copied from the original document.
+
+        :raises RuntimeError: If the component wasn't warmed up.
+        :raises TypeError: If the input is not a list of Documents.
+        :raises ValueError: If the document content is None or empty.
+        """
+        self.warm_up()
+
+        if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
+            raise TypeError("EmbeddingBasedDocumentSplitter expects a List of Documents as input.")
+
+        split_docs: list[Document] = []
+        for doc in documents:
+            if doc.content is None:
+                raise ValueError(
+                    f"EmbeddingBasedDocumentSplitter only works with text documents but content for "
+                    f"document ID {doc.id} is None."
+                )
+            if doc.content == "":
+                logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
+                continue
+
+            doc_splits = self._split_document(doc=doc)
+            split_docs.extend(doc_splits)
+
+        return {"documents": split_docs}
+
+    @component.output_types(documents=list[Document])
+    async def run_async(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Asynchronously split documents based on embedding similarity.
+
+        This is the asynchronous version of the `run` method with the same parameters and return values.
+
+        :param documents: The documents to split.
+        :returns: A dictionary with the following key:
+            - `documents`: List of documents with the split texts. Each document includes:
+                - A metadata field `source_id` to track the original document.
+                - A metadata field `split_id` to track the split number.
+                - A metadata field `page_number` to track the original page number.
+                - All other metadata copied from the original document.
+
+        :raises RuntimeError: If the component wasn't warmed up.
+        :raises TypeError: If the input is not a list of Documents.
+        :raises ValueError: If the document content is None or empty.
+        """
+        await self.warm_up_async()
+
+        if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
+            raise TypeError("EmbeddingBasedDocumentSplitter expects a List of Documents as input.")
+
+        tasks: list[Awaitable[list[Document]]] = []
+        for doc in documents:
+            if doc.content is None:
+                raise ValueError(
+                    f"EmbeddingBasedDocumentSplitter only works with text documents but content for "
+                    f"document ID {doc.id} is None."
+                )
+            if doc.content == "":
+                logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
+                continue
+
+            tasks.append(self._split_document_async(doc=doc))
+
+        return {"documents": [*chain.from_iterable(await gather(*tasks))]}
+
+    def _split_document(self, doc: Document) -> list[Document]:
+        """
+        Split a single document based on embedding similarity.
+        """
+        # Create an initial split of the document content into smaller chunks
+        # doc.content is validated in `run`
+        splits = self._split_text(text=doc.content)  # type: ignore[arg-type]
+
+        # Merge splits smaller than min_length
+        merged_splits = self._merge_small_splits(splits=splits)
+
+        # Recursively split splits larger than max_length
+        final_splits = self._split_large_splits(splits=merged_splits)
+
+        # Create Document objects from the final splits
+        return EmbeddingBasedDocumentSplitter._create_documents_from_splits(splits=final_splits, original_doc=doc)
+
+    async def _split_document_async(self, doc: Document) -> list[Document]:
+        """
+        Split a single document based on embedding similarity.
+        """
+        # Create an initial split of the document content into smaller chunks
+        # doc.content is validated in `run`
+        splits = await self._split_text_async(text=doc.content)  # type: ignore[arg-type]
+
+        # Merge splits smaller than min_length
+        merged_splits = self._merge_small_splits(splits=splits)
+
+        # Recursively split splits larger than max_length
+        final_splits = self._split_large_splits(splits=merged_splits)
+
+        # Create Document objects from the final splits
+        return EmbeddingBasedDocumentSplitter._create_documents_from_splits(splits=final_splits, original_doc=doc)
+
+    def _prepare_sentence_groups(self, text: str) -> list[str]:
+        """Preprocess raw text into grouped sentences ready for embedding."""
+        # NOTE: `self.sentence_splitter.split_sentences` strips all white space types (e.g. new lines, page breaks,
+        # etc.) at the end of the provided text. So to not lose them, we need keep track of them and add them back to
+        # the last sentence.
+        rstripped_text = text.rstrip()
+        trailing_whitespaces = text[len(rstripped_text) :]
+
+        # Split the text into sentences
+        sentences_result = self.sentence_splitter.split_sentences(rstripped_text)  # type: ignore[union-attr]
+
+        # Add back the stripped white spaces to the last sentence
+        if sentences_result and trailing_whitespaces:
+            sentences_result[-1]["sentence"] += trailing_whitespaces
+            sentences_result[-1]["end"] += len(trailing_whitespaces)
+
+        sentences = [sentence["sentence"] for sentence in sentences_result]
+        return self._group_sentences(sentences=sentences)
+
+    def _split_text(self, text: str) -> list[str]:
+        """
+        Split a text into smaller chunks based on embedding similarity.
+        """
+        sentence_groups = self._prepare_sentence_groups(text=text)
+        embeddings = self._calculate_embeddings(sentence_groups=sentence_groups)
+        split_points = self._find_split_points(embeddings=embeddings)
+        return self._create_splits_from_points(sentence_groups=sentence_groups, split_points=split_points)
+
+    async def _split_text_async(self, text: str) -> list[str]:
+        """
+        Asynchronously split a text into smaller chunks based on embedding similarity.
+        """
+        sentence_groups = self._prepare_sentence_groups(text=text)
+        embeddings = await self._calculate_embeddings_async(sentence_groups=sentence_groups)
+        split_points = self._find_split_points(embeddings=embeddings)
+        return self._create_splits_from_points(sentence_groups=sentence_groups, split_points=split_points)
+
+    def _group_sentences(self, sentences: list[str]) -> list[str]:
+        """
+        Group sentences into groups of sentences_per_group.
+        """
+        if self.sentences_per_group == 1:
+            return sentences
+
+        groups = []
+        for i in range(0, len(sentences), self.sentences_per_group):
+            group = sentences[i : i + self.sentences_per_group]
+            groups.append("".join(group))
+
+        return groups
+
+    def _calculate_embeddings(self, sentence_groups: list[str]) -> list[list[float]]:
+        """
+        Calculate embeddings for each sentence group using the DocumentEmbedder.
+        """
+        # Create Document objects for each group
+        group_docs = [Document(content=group) for group in sentence_groups]
+        result = self.document_embedder.run(group_docs)
+        embedded_docs = result["documents"]
+        return [doc.embedding for doc in embedded_docs]
+
+    async def _calculate_embeddings_async(self, sentence_groups: list[str]) -> list[list[float]]:
+        """
+        Asynchronously Calculate embeddings for each sentence group using the DocumentEmbedder.
+        """
+        # Create Document objects for each group
+        group_docs = [Document(content=group) for group in sentence_groups]
+        result = await _execute_component_async(self.document_embedder, documents=group_docs)
+        embedded_docs = result["documents"]
+        return [doc.embedding for doc in embedded_docs]
+
+    def _find_split_points(self, embeddings: list[list[float]]) -> list[int]:
+        """
+        Find split points based on cosine distances between sequential embeddings.
+        """
+        if len(embeddings) <= 1:
+            return []
+
+        # Calculate cosine distances between sequential pairs
+        distances = []
+        for i in range(len(embeddings) - 1):
+            distance = EmbeddingBasedDocumentSplitter._cosine_distance(
+                embedding1=embeddings[i], embedding2=embeddings[i + 1]
+            )
+            distances.append(distance)
+
+        # Calculate threshold based on percentile
+        threshold = np.percentile(distances, self.percentile * 100)
+
+        # Find indices where distance exceeds threshold
+        split_points = []
+        for i, distance in enumerate(distances):
+            if distance > threshold:
+                split_points.append(i + 1)  # +1 because we want to split after this point
+
+        return split_points
+
+    @staticmethod
+    def _cosine_distance(embedding1: list[float], embedding2: list[float]) -> float:
+        """
+        Calculate cosine distance between two embeddings.
+        """
+        vec1 = np.array(embedding1)
+        vec2 = np.array(embedding2)
+
+        norm1 = float(np.linalg.norm(vec1))
+        norm2 = float(np.linalg.norm(vec2))
+
+        if norm1 == 0 or norm2 == 0:
+            return 1.0
+
+        cosine_sim = float(np.dot(vec1, vec2) / (norm1 * norm2))
+
+        return 1.0 - cosine_sim
+
+    @staticmethod
+    def _create_splits_from_points(sentence_groups: list[str], split_points: list[int]) -> list[str]:
+        """
+        Create splits based on split points.
+        """
+        if not split_points:
+            return ["".join(sentence_groups)]
+
+        splits = []
+        start = 0
+
+        for point in split_points:
+            split_text = "".join(sentence_groups[start:point])
+            if split_text:
+                splits.append(split_text)
+            start = point
+
+        # Add the last split
+        if start < len(sentence_groups):
+            split_text = "".join(sentence_groups[start:])
+            if split_text:
+                splits.append(split_text)
+
+        return splits
+
+    def _merge_small_splits(self, splits: list[str]) -> list[str]:
+        """
+        Merge splits that are below min_length.
+        """
+        if not splits:
+            return splits
+
+        merged = []
+        current_split = splits[0]
+
+        for split in splits[1:]:
+            # We merge splits that are smaller than min_length but only if the newly merged split is still below
+            # max_length.
+            if len(current_split) < self.min_length and len(current_split) + len(split) < self.max_length:
+                # Merge with next split
+                current_split += split
+            else:
+                # Current split is long enough, save it and start a new one
+                merged.append(current_split)
+                current_split = split
+
+        # Don't forget the last split
+        merged.append(current_split)
+
+        return merged
+
+    def _split_large_splits(self, splits: list[str]) -> list[str]:
+        """
+        Recursively split splits that are above max_length.
+
+        This method checks each split and if it exceeds max_length, it attempts to split it further using the same
+        embedding-based approach. This is done recursively until all splits are within the max_length limit or no
+        further splitting is possible.
+
+        This works because the threshold for splits is calculated dynamically based on the provided of embeddings.
+        """
+        final_splits = []
+
+        for split in splits:
+            if len(split) <= self.max_length:
+                final_splits.append(split)
+            else:
+                # Recursively split large splits
+                # We can reuse the same _split_text method to split the text into smaller chunks because the threshold
+                # for splits is calculated dynamically based on embeddings from `split`.
+                sub_splits = self._split_text(text=split)
+
+                # Stop splitting if no further split is possible or continue with recursion
+                if len(sub_splits) == 1:
+                    logger.warning(
+                        "Could not split a chunk further below max_length={max_length}. "
+                        "Returning chunk of length {length}.",
+                        max_length=self.max_length,
+                        length=len(split),
+                    )
+                    final_splits.append(split)
+                else:
+                    final_splits.extend(self._split_large_splits(splits=sub_splits))
+
+        return final_splits
+
+    @staticmethod
+    def _create_documents_from_splits(splits: list[str], original_doc: Document) -> list[Document]:
+        """
+        Create Document objects from splits.
+        """
+        documents = []
+        metadata = deepcopy(original_doc.meta)
+        metadata["source_id"] = original_doc.id
+
+        # Calculate page numbers for each split
+        current_page = 1
+
+        for i, split_text in enumerate(splits):
+            split_meta = deepcopy(metadata)
+            split_meta["split_id"] = i
+
+            # Calculate page number for this split
+            # Count page breaks in the split itself
+            page_breaks_in_split = split_text.count("\f")
+
+            # Calculate the page number for this split
+            split_meta["page_number"] = current_page
+
+            doc = Document(content=split_text, meta=split_meta)
+            documents.append(doc)
+
+            # Update page counter for next split
+            current_page += page_breaks_in_split
+
+        return documents
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Serialized dictionary representation of the component.
+        """
+        return default_to_dict(
+            self,
+            document_embedder=component_to_dict(obj=self.document_embedder, name="document_embedder"),
+            sentences_per_group=self.sentences_per_group,
+            percentile=self.percentile,
+            min_length=self.min_length,
+            max_length=self.max_length,
+            language=self.language,
+            use_split_rules=self.use_split_rules,
+            extend_abbreviations=self.extend_abbreviations,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "EmbeddingBasedDocumentSplitter":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize and create the component.
+
+        :returns:
+            The deserialized component.
+        """
+        deserialize_component_inplace(data["init_parameters"], key="document_embedder")
+        return default_from_dict(cls, data)

haystack/components/preprocessors/hierarchical_document_splitter.py

@@ -0,0 +1,156 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from dataclasses import replace
+from typing import Any, Literal
+
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.components.preprocessors import DocumentSplitter
+
+
+@component
+class HierarchicalDocumentSplitter:
+    """
+    Splits a documents into different block sizes building a hierarchical tree structure of blocks of different sizes.
+
+    The root node of the tree is the original document, the leaf nodes are the smallest blocks. The blocks in between
+    are connected such that the smaller blocks are children of the parent-larger blocks.
+
+    ## Usage example
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import HierarchicalDocumentSplitter
+
+    doc = Document(content="This is a simple test document")
+    splitter = HierarchicalDocumentSplitter(block_sizes={3, 2}, split_overlap=0, split_by="word")
+    splitter.run([doc])
+    # >> {'documents': [Document(id=3f7..., content: 'This is a simple test document', meta: {'block_size': 0, 'parent_id': None, 'children_ids': ['5ff..', '8dc..'], 'level': 0}),
+    # >> Document(id=5ff.., content: 'This is a ', meta: {'block_size': 3, 'parent_id': '3f7..', 'children_ids': ['f19..', '52c..'], 'level': 1, 'source_id': '3f7..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
+    # >> Document(id=8dc.., content: 'simple test document', meta: {'block_size': 3, 'parent_id': '3f7..', 'children_ids': ['39d..', 'e23..'], 'level': 1, 'source_id': '3f7..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 10}),
+    # >> Document(id=f19.., content: 'This is ', meta: {'block_size': 2, 'parent_id': '5ff..', 'children_ids': [], 'level': 2, 'source_id': '5ff..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
+    # >> Document(id=52c.., content: 'a ', meta: {'block_size': 2, 'parent_id': '5ff..', 'children_ids': [], 'level': 2, 'source_id': '5ff..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 8}),
+    # >> Document(id=39d.., content: 'simple test ', meta: {'block_size': 2, 'parent_id': '8dc..', 'children_ids': [], 'level': 2, 'source_id': '8dc..', 'page_number': 1, 'split_id': 0, 'split_idx_start': 0}),
+    # >> Document(id=e23.., content: 'document', meta: {'block_size': 2, 'parent_id': '8dc..', 'children_ids': [], 'level': 2, 'source_id': '8dc..', 'page_number': 1, 'split_id': 1, 'split_idx_start': 12})]}
+    ```
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        block_sizes: set[int],
+        split_overlap: int = 0,
+        split_by: Literal["word", "sentence", "page", "passage"] = "word",
+    ) -> None:
+        """
+        Initialize HierarchicalDocumentSplitter.
+
+        :param block_sizes: Set of block sizes to split the document into. The blocks are split in descending order.
+        :param split_overlap: The number of overlapping units for each split.
+        :param split_by: The unit for splitting your documents.
+        :raises ValueError: If `block_sizes` is empty, if `split_overlap` is negative, or if `split_overlap` is
+            greater than or equal to the smallest value in `block_sizes`.
+        """
+
+        if not block_sizes:
+            raise ValueError("block_sizes must not be empty. Provide at least one block size.")
+
+        if split_overlap < 0:
+            raise ValueError("split_overlap must be greater than or equal to 0.")
+
+        smallest_block_size = min(block_sizes)
+        if split_overlap >= smallest_block_size:
+            raise ValueError(
+                f"split_overlap ({split_overlap}) must be less than the smallest value in block_sizes "
+                f"({smallest_block_size}). Reduce split_overlap or increase the smallest block size."
+            )
+
+        self.block_sizes = sorted(set(block_sizes), reverse=True)
+        self.splitters: dict[int, DocumentSplitter] = {}
+        self.split_overlap = split_overlap
+        self.split_by = split_by
+        self._build_block_sizes()
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Builds a hierarchical document structure for each document in a list of documents.
+
+        :param documents: List of Documents to split into hierarchical blocks.
+        :returns: List of HierarchicalDocument
+        """
+        hierarchical_docs = []
+        for doc in documents:
+            hierarchical_docs.extend(self.build_hierarchy_from_doc(doc))
+        return {"documents": hierarchical_docs}
+
+    def _build_block_sizes(self) -> None:
+        for block_size in self.block_sizes:
+            self.splitters[block_size] = DocumentSplitter(
+                split_length=block_size, split_overlap=self.split_overlap, split_by=self.split_by
+            )
+
+    @staticmethod
+    def _add_meta_data(document: Document) -> Document:
+        new_meta = {**document.meta, "__block_size": 0, "__parent_id": None, "__children_ids": [], "__level": 0}
+        return replace(document, meta=new_meta)
+
+    def build_hierarchy_from_doc(self, document: Document) -> list[Document]:
+        """
+        Build a hierarchical tree document structure from a single document.
+
+        Given a document, this function splits the document into hierarchical blocks of different sizes represented
+        as HierarchicalDocument objects.
+
+        :param document: Document to split into hierarchical blocks.
+        :returns:
+            List of HierarchicalDocument
+        """
+
+        root = self._add_meta_data(document)
+        current_level_nodes = [root]
+        all_docs = []
+
+        for block in self.block_sizes:
+            next_level_nodes = []
+            for doc in current_level_nodes:
+                splitted_docs = self.splitters[block].run([doc])
+                child_docs = splitted_docs["documents"]
+                # if it's only one document skip
+                if len(child_docs) == 1:
+                    next_level_nodes.append(doc)
+                    continue
+                for child_doc in child_docs:
+                    child_doc = self._add_meta_data(child_doc)
+                    child_doc.meta["__level"] = doc.meta["__level"] + 1
+                    child_doc.meta["__block_size"] = block
+                    child_doc.meta["__parent_id"] = doc.id
+                    all_docs.append(child_doc)
+                    doc.meta["__children_ids"].append(child_doc.id)
+                    next_level_nodes.append(child_doc)
+            current_level_nodes = next_level_nodes
+
+        return [root] + all_docs
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Returns a dictionary representation of the component.
+
+        :returns:
+                Serialized dictionary representation of the component.
+        """
+        return default_to_dict(
+            self, block_sizes=self.block_sizes, split_overlap=self.split_overlap, split_by=self.split_by
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "HierarchicalDocumentSplitter":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize and create the component.
+
+        :returns:
+            The deserialized component.
+        """
+        return default_from_dict(cls, data)

haystack/components/preprocessors/markdown_header_splitter.py

@@ -0,0 +1,382 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from typing import Literal
+
+from haystack import Document, component, logging
+from haystack.components.preprocessors import DocumentSplitter
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class MarkdownHeaderSplitter:
+    """
+    Split documents at ATX-style Markdown headers (#), with optional secondary splitting.
+
+    This component processes text documents by:
+    - Splitting them into chunks at Markdown headers (e.g., '#', '##', etc.), preserving header hierarchy as metadata.
+    - Optionally applying a secondary split (by word, passage, period, or line) to each chunk
+      (using haystack's DocumentSplitter).
+    - Preserving and propagating metadata such as parent headers, page numbers, and split IDs.
+    """
+
+    def __init__(
+        self,
+        *,
+        page_break_character: str = "\f",
+        keep_headers: bool = True,
+        header_split_levels: list[int] | None = None,
+        secondary_split: Literal["word", "passage", "period", "line"] | None = None,
+        split_length: int = 200,
+        split_overlap: int = 0,
+        split_threshold: int = 0,
+        skip_empty_documents: bool = True,
+    ) -> None:
+        """
+        Initialize the MarkdownHeaderSplitter.
+
+        :param page_break_character: Character used to identify page breaks. Defaults to form feed ("\f").
+        :param keep_headers: If True, headers are kept in the content. If False, headers are moved to metadata.
+            Defaults to True.
+        :param header_split_levels: List of header levels (1–6) to split on. For example, `[1, 2]` splits only
+            on `#` and `##` headers, merging content under deeper headers into the preceding chunk. Defaults to
+            all levels `[1, 2, 3, 4, 5, 6]`.
+        :param secondary_split: Optional secondary split condition after header splitting.
+            Options are None, "word", "passage", "period", "line". Defaults to None.
+        :param split_length: The maximum number of units in each split when using secondary splitting. Defaults to 200.
+        :param split_overlap: The number of overlapping units for each split when using secondary splitting.
+            Defaults to 0.
+        :param split_threshold: The minimum number of units per split when using secondary splitting. Defaults to 0.
+        :param skip_empty_documents: Choose whether to skip documents with empty content. Default is True.
+            Set to False when downstream components in the Pipeline (like LLMDocumentContentExtractor) can extract text
+            from non-textual documents.
+        """
+        if header_split_levels is None:
+            header_split_levels = [1, 2, 3, 4, 5, 6]
+
+        if not isinstance(header_split_levels, list) or len(header_split_levels) == 0:
+            raise ValueError("header_split_levels must be a non-empty list.")
+        invalid = [lvl for lvl in header_split_levels if not isinstance(lvl, int) or lvl < 1 or lvl > 6]
+        if invalid:
+            raise ValueError(
+                f"header_split_levels contains invalid values: {invalid}. All levels must be integers between 1 and 6."
+            )
+        if len(header_split_levels) != len(set(header_split_levels)):
+            raise ValueError("header_split_levels must not contain duplicate values.")
+
+        self.page_break_character = page_break_character
+        self.secondary_split = secondary_split
+        self.split_length = split_length
+        self.split_overlap = split_overlap
+        self.split_threshold = split_threshold
+        self.skip_empty_documents = skip_empty_documents
+        self.keep_headers = keep_headers
+        self.header_split_levels = header_split_levels
+        self._header_split_levels_set = set(header_split_levels)
+        self._header_pattern = re.compile(r"(?m)^(#{1,6}) (.+)$")  # ATX-style .md-headers
+
+        # Matches fenced code blocks delimited by triple backticks (```) or triple tildes (~~~).
+        # Broken down:
+        #   ^                 - fence must start at the beginning of a line (MULTILINE)
+        #   (?P<fence>`{3,}|~{3,})
+        #                     - named capture group "fence": three or more backticks OR three or
+        #                       more tildes. Capturing it allows the closing fence to be matched
+        #                       with a backreference, so ```-opened blocks must close with ```
+        #                       and ~~~-opened blocks must close with ~~~.
+        #   [^\n]*            - optional language identifier (e.g. "python") and any other text
+        #                       on the opening fence line, up to the newline
+        #   \n                - newline ending the opening fence line
+        #   .*?               - the code block body, matched lazily (DOTALL so . matches newlines)
+        #   ^(?P=fence)       - closing fence: must be identical to the opening fence (backreference),
+        #                       and must start at the beginning of a line
+        #   \s*$              - optional trailing whitespace after the closing fence
+        self._code_block_pattern = re.compile(
+            r"^(?P<fence>`{3,}|~{3,})[^\n]*\n.*?^(?P=fence)\s*$", re.MULTILINE | re.DOTALL
+        )
+
+        self._is_warmed_up = False
+
+        # initialize secondary_splitter only if needed
+        if self.secondary_split:
+            self.secondary_splitter = DocumentSplitter(
+                split_by=self.secondary_split,
+                split_length=self.split_length,
+                split_overlap=self.split_overlap,
+                split_threshold=self.split_threshold,
+            )
+
+    def warm_up(self) -> None:
+        """
+        Warm up the MarkdownHeaderSplitter.
+        """
+        if self.secondary_split and not self._is_warmed_up:
+            self.secondary_splitter.warm_up()
+            self._is_warmed_up = True
+
+    def _code_block_spans(self, text: str) -> list[tuple[int, int]]:
+        """Return the (start, end) character spans of all fenced code blocks in text."""
+        return [(m.start(), m.end()) for m in self._code_block_pattern.finditer(text)]
+
+    def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
+        """Split text by ATX-style headers (#) and create chunks with appropriate metadata."""
+        logger.debug("Splitting text by markdown headers")
+
+        # Pre-compute fenced code block spans so that # lines inside code blocks (e.g. Python comments) are not
+        # mistaken for Markdown headers.
+        code_spans = self._code_block_spans(text)
+
+        # find headers at the configured levels only, excluding any that fall inside a code block. Content between
+        # skipped headers is absorbed into the preceding chunk's span since end = next_match.start().
+        matches = [
+            m
+            for m in re.finditer(self._header_pattern, text)
+            if len(m.group(1)) in self._header_split_levels_set
+            and not any(start <= m.start() < end for start, end in code_spans)
+        ]
+
+        # return unsplit if no headers found
+        if not matches:
+            logger.info(
+                "No headers found in document {doc_id}; returning full document as single chunk.", doc_id=doc_id
+            )
+            return [{"content": text, "meta": {}}]
+
+        # process headers and build chunks
+        chunks: list[dict] = []
+        header_stack: list[str | None] = [None] * 6
+        pending_headers: list[str] = []  # store empty headers to prepend to next content
+        has_content = False  # flag to track if any header has content
+
+        for i, match in enumerate(matches):
+            # extract header info
+            header_prefix = match.group(1)
+            header_text = match.group(2)
+            level = len(header_prefix)
+
+            # get content
+            start = match.end()
+            end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+            content = text[start:end]
+
+            # update header stack to track nesting
+            header_stack[level - 1] = header_text
+            for j in range(level, 6):
+                header_stack[j] = None
+
+            # skip splits w/o content
+            if not content.strip():  # this strip is needed to avoid counting whitespace as content
+                if self.keep_headers:
+                    header_line = f"{header_prefix} {header_text}"
+                    pending_headers.append(header_line)
+                continue
+
+            has_content = True  # at least one header has content
+            # Build parent metadata from the current header stack so the first child of a
+            # contentful section still inherits its full ancestor chain.
+            parent_headers = [h for h in header_stack[: level - 1] if h is not None]
+
+            logger.debug(
+                "Creating chunk for header '{header_text}' at level {level}", header_text=header_text, level=level
+            )
+
+            if self.keep_headers:
+                header_line = f"{header_prefix} {header_text}"
+                # add pending & current header to content
+                chunk_content = ""
+                if pending_headers:
+                    chunk_content += "\n".join(pending_headers) + "\n"
+                chunk_content += f"{header_line}{content}"
+                chunks.append(
+                    {"content": chunk_content, "meta": {"header": header_text, "parent_headers": parent_headers}}
+                )
+                pending_headers = []  # reset pending headers
+            else:
+                chunks.append({"content": content, "meta": {"header": header_text, "parent_headers": parent_headers}})
+
+        # return doc unchunked if no headers have content
+        if not has_content:
+            logger.info(
+                "Document {doc_id} contains only headers with no content; returning original document.", doc_id=doc_id
+            )
+            return [{"content": text, "meta": {}}]
+
+        return chunks
+
+    def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document]:
+        """
+        Apply secondary splitting while preserving header metadata and structure.
+
+        Ensures page counting is maintained across splits.
+        """
+        result_docs = []
+        current_split_id = 0  # track split_id across all secondary splits from the same parent
+
+        for doc in documents:
+            if doc.content is None:
+                result_docs.append(doc)
+                continue
+
+            content_for_splitting: str = doc.content
+
+            if not self.keep_headers:  # skip header extraction if keep_headers
+                # extract header information
+                header_match = re.match(self._header_pattern, doc.content)
+                if header_match:
+                    content_for_splitting = doc.content[header_match.end() :]
+
+            # track page from meta
+            current_page = doc.meta.get("page_number", 1)
+
+            # create a clean meta dict without split_id for secondary splitting
+            clean_meta = {k: v for k, v in doc.meta.items() if k != "split_id"}
+
+            secondary_splits = self.secondary_splitter.run(
+                documents=[Document(content=content_for_splitting, meta=clean_meta)]
+            )["documents"]
+
+            # split processing
+            for i, split in enumerate(secondary_splits):
+                # calculate page number for this split
+                if i > 0 and secondary_splits[i - 1].content:
+                    current_page = self._update_page_number_with_breaks(secondary_splits[i - 1].content, current_page)
+
+                # set page number and split_id to meta
+                split.meta["page_number"] = current_page
+                split.meta["split_id"] = current_split_id
+                # ensure source_id is preserved from the original document
+                if "source_id" in doc.meta:
+                    split.meta["source_id"] = doc.meta["source_id"]
+                current_split_id += 1
+
+                # preserve header metadata if we're not keeping headers in content
+                if not self.keep_headers:
+                    for key in ["header", "parent_headers"]:
+                        if key in doc.meta:
+                            split.meta[key] = doc.meta[key]
+
+                result_docs.append(split)
+
+        logger.debug(
+            "Secondary splitting complete. Final count: {final_count} documents.", final_count=len(result_docs)
+        )
+        return result_docs
+
+    def _update_page_number_with_breaks(self, content: str | None, current_page: int) -> int:
+        """
+        Update page number based on page breaks in content.
+
+        :param content: Content to check for page breaks
+        :param current_page: Current page number
+        :return: New current page number
+        """
+        if not isinstance(content, str):
+            return current_page
+
+        page_breaks = content.count(self.page_break_character)
+        new_page_number = current_page + page_breaks
+
+        if page_breaks > 0:
+            logger.debug(
+                "Found {page_breaks} page breaks, page number updated: {old} → {new}",
+                page_breaks=page_breaks,
+                old=current_page,
+                new=new_page_number,
+            )
+
+        return new_page_number
+
+    def _split_documents_by_markdown_headers(self, documents: list[Document]) -> list[Document]:
+        """Split a list of documents by markdown headers, preserving metadata."""
+
+        result_docs = []
+        for doc in documents:
+            logger.debug("Splitting document with id={doc_id}", doc_id=doc.id)
+            # mypy: doc.content is Optional[str], so we must check for None before passing to splitting method
+            if doc.content is None:
+                continue
+            splits = self._split_text_by_markdown_headers(doc.content, doc.id)
+            docs = []
+
+            current_page = doc.meta.get("page_number", 1) if doc.meta else 1
+            total_page_breaks = doc.content.count(self.page_break_character)
+            logger.debug(
+                "Processing document with id={doc_id}: starting at page {start_page}, "
+                "contains {page_breaks} page breaks in total",
+                doc_id=doc.id,
+                start_page=current_page,
+                page_breaks=total_page_breaks,
+            )
+            for split_idx, split in enumerate(splits):
+                meta = doc.meta.copy() if doc.meta else {}
+                meta.update({"source_id": doc.id, "page_number": current_page, "split_id": split_idx})
+                if split.get("meta"):
+                    meta.update(split["meta"])
+                current_page = self._update_page_number_with_breaks(split["content"], current_page)
+                docs.append(Document(content=split["content"], meta=meta))
+            logger.debug(
+                "Split into {num_docs} documents for id={doc_id}, final page: {current_page}",
+                num_docs=len(docs),
+                doc_id=doc.id,
+                current_page=current_page,
+            )
+            result_docs.extend(docs)
+        return result_docs
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Run the markdown header splitter with optional secondary splitting.
+
+        :param documents: List of documents to split
+
+        :returns: A dictionary with the following key:
+            - `documents`: List of documents with the split texts. Each document includes:
+                - A metadata field `source_id` to track the original document.
+                - A metadata field `page_number` to track the original page number.
+                - A metadata field `split_id` to identify the split chunk index within its parent document.
+                - All other metadata copied from the original document.
+        :raises ValueError: If a document has `None` content.
+        :raises TypeError: If a document's content is not a string.
+        """
+        if self.secondary_split and not self._is_warmed_up:
+            self.warm_up()
+        # validate input documents
+        for doc in documents:
+            if doc.content is None:
+                raise ValueError(
+                    "MarkdownHeaderSplitter only works with text documents but content for document ID"
+                    f" {doc.id} is None."
+                )
+            if not isinstance(doc.content, str):
+                raise TypeError("MarkdownHeaderSplitter only works with text documents (str content).")
+
+        final_docs = []
+        for doc in documents:
+            # handle empty documents
+            if not doc.content or not doc.content.strip():  # avoid counting whitespace as content
+                if self.skip_empty_documents:
+                    logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
+                    continue
+                # keep empty documents
+                final_docs.append(doc)
+                logger.warning(
+                    "Document ID {doc_id} has an empty content. Keeping this document as per configuration.",
+                    doc_id=doc.id,
+                )
+                continue
+
+            # split this document by headers
+            header_split_docs = self._split_documents_by_markdown_headers([doc])
+
+            # apply secondary splitting if configured
+            if self.secondary_split:
+                doc_splits = self._apply_secondary_splitting(header_split_docs)
+            else:
+                doc_splits = header_split_docs
+
+            final_docs.extend(doc_splits)
+
+        return {"documents": final_docs}

haystack/components/preprocessors/python_code_splitter.py

@@ -0,0 +1,612 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import ast
+import math
+from dataclasses import dataclass, field
+from typing import Any
+
+from haystack import Document, component, logging
+from haystack.components.preprocessors.document_splitter import DocumentSplitter
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class _CodeUnit:
+    """One syntactic split unit (function, class header, method, imports block, statement, ...)."""
+
+    source: str
+    start_line: int
+    end_line: int
+    kind: str
+    name: str | None = None
+    class_name: str | None = None
+    class_signature: str | None = None
+    decorators: list[str] = field(default_factory=list)
+    docstring: str | None = None
+
+
+@component
+class PythonCodeSplitter:
+    """
+    Split Python source code into syntax-aware chunks.
+
+    The component parses each source with :mod:`ast` into *units* (module docstring,
+    consecutive ``import`` blocks, top-level functions, class headers, methods, nested
+    classes, and remaining statements) and merges them greedily in source order toward
+    ``max_effective_lines`` per chunk, where effective lines are
+    ``ceil(len(source) / expected_chars_per_line)``. Functions and methods are kept
+    whole; the resulting chunks read top-to-bottom like the original file with comments
+    and blank lines preserved.
+
+    A function whose effective length exceeds ``oversized_factor * max_effective_lines``
+    is the only case where chunks may overlap: it is broken down with a line-based
+    secondary split (:class:`DocumentSplitter`, ``split_by="line"``) and the resulting
+    pieces carry ``secondary_split=True`` along with the originating function's metadata.
+    The primary split never adds overlap.
+
+    Per-chunk metadata: ``source_id``, ``split_id``, ``start_line``, ``end_line``,
+    ``unit_kinds``; plus ``include_classes``, ``decorators``, and ``docstrings`` (when
+    ``strip_docstrings=True``) where applicable. ``file_name`` and any other parent
+    document meta are propagated.
+
+    Usage example:
+
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import PythonCodeSplitter
+
+    source = '''
+    \"\"\"Example module.\"\"\"
+    from math import sqrt
+
+
+    class Circle:
+        def __init__(self, r: float) -> None:
+            self.r = r
+
+        def area(self) -> float:
+            return 3.14159 * self.r * self.r
+    '''
+
+    splitter = PythonCodeSplitter(min_effective_lines=4, max_effective_lines=6)
+    result = splitter.run(documents=[Document(content=source, meta={"file_name": "circle.py"})])
+    for chunk in result["documents"]:
+        print(chunk.meta["start_line"], chunk.meta["end_line"], chunk.meta.get("include_classes"))
+    ```
+
+    Pass ``strip_docstrings=True`` to move docstrings out of the chunk content and into
+    each chunk's ``meta["docstrings"]`` list. This is useful for RAG when docstrings are
+    large: stripping shrinks the stored content while the docstring text can still
+    influence retrieval via ``meta_fields_to_embed=["docstrings"]`` on the embedder.
+    """
+
+    def __init__(
+        self,
+        *,
+        min_effective_lines: int = 20,
+        max_effective_lines: int = 100,
+        expected_chars_per_line: int = 45,
+        oversized_factor: int = 3,
+        strip_docstrings: bool = False,
+        preserve_class_definition: bool = True,
+        secondary_split_overlap: int = 5,
+        secondary_split_length: int | None = None,
+    ) -> None:
+        """
+        Initialize the PythonCodeSplitter.
+
+        :param min_effective_lines: Minimum effective lines per chunk. While the running
+            chunk is below this threshold the splitter keeps merging in the next unit.
+        :param max_effective_lines: Target effective lines per chunk. Units are merged
+            greedily while doing so brings the running total closer to this target.
+        :param expected_chars_per_line: Used to convert characters into effective lines as
+            ``ceil(len(source) / expected_chars_per_line)``; long lines count as more than one.
+        :param oversized_factor: A function whose effective length exceeds
+            ``oversized_factor * max_effective_lines`` triggers the line-based secondary
+            split with overlap.
+        :param strip_docstrings: If ``True``, function/method/class docstrings are moved
+            from the chunk content into ``meta["docstrings"]`` (source order). The
+            module-level docstring is kept in place since it is itself a top-level unit.
+        :param preserve_class_definition: If ``True`` (default), chunks that contain class
+            members but not the class header are prefixed with the bare class signature
+            (decorators plus the ``class Foo(...):`` lines) in source order.
+        :param secondary_split_overlap: Line overlap for the secondary splitter; only used
+            in the oversized fallback. The primary AST split never adds overlap.
+        :param secondary_split_length: Lines per chunk for the secondary splitter.
+            Defaults to ``max_effective_lines`` when ``None``.
+        :raises ValueError: If any parameter is invalid (negative, zero where positive is
+            required, or ``min_effective_lines > max_effective_lines``).
+        """
+        if min_effective_lines < 1:
+            raise ValueError("min_effective_lines must be at least 1.")
+        if max_effective_lines < 1:
+            raise ValueError("max_effective_lines must be at least 1.")
+        if min_effective_lines > max_effective_lines:
+            raise ValueError("min_effective_lines must not be greater than max_effective_lines.")
+        if expected_chars_per_line < 1:
+            raise ValueError("expected_chars_per_line must be at least 1.")
+        if oversized_factor < 1:
+            raise ValueError("oversized_factor must be at least 1.")
+        if secondary_split_overlap < 0:
+            raise ValueError("secondary_split_overlap must be non-negative.")
+        if secondary_split_length is not None and secondary_split_length < 1:
+            raise ValueError("secondary_split_length must be at least 1.")
+
+        self.min_effective_lines = min_effective_lines
+        self.max_effective_lines = max_effective_lines
+        self.expected_chars_per_line = expected_chars_per_line
+        self.oversized_factor = oversized_factor
+        self.strip_docstrings = strip_docstrings
+        self.preserve_class_definition = preserve_class_definition
+        self.secondary_split_overlap = secondary_split_overlap
+        self.secondary_split_length = secondary_split_length
+
+    def _effective_lines(self, text: str) -> int:
+        """Return the number of *effective lines* for ``text`` (see class docstring)."""
+        if not text:
+            return 0
+        return max(1, math.ceil(len(text) / self.expected_chars_per_line))
+
+    def _is_oversized(self, unit: "_CodeUnit") -> bool:
+        """Return ``True`` if ``unit`` should trigger the secondary line-based split."""
+        return self._effective_lines(unit.source) > self.oversized_factor * self.max_effective_lines
+
+    @staticmethod
+    def _slice_lines(source_lines: list[str], start: int, end: int) -> str:
+        """Slice ``source_lines`` between the 1-indexed ``start`` and ``end`` (inclusive)."""
+        start = max(start, 1)
+        if end < start:
+            return ""
+        return "".join(source_lines[start - 1 : end])
+
+    @staticmethod
+    def _safe_unparse(node: ast.AST) -> str:
+        """Return ``ast.unparse(node)`` but tolerate exotic nodes by falling back to ``repr``."""
+        try:
+            return ast.unparse(node)
+        except Exception:  # pragma: no cover - defensive guard
+            return repr(node)
+
+    def _strip_docstring(
+        self,
+        node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef,
+        source_lines: list[str],
+        unit_start: int,
+        unit_end: int,
+    ) -> tuple[str, str | None]:
+        """Strip ``node``'s docstring from ``source_lines[unit_start..unit_end]`` if safely possible."""
+        docstring = ast.get_docstring(node)
+        body = node.body
+        if not docstring or not body:
+            return self._slice_lines(source_lines, unit_start, unit_end), None
+
+        first = body[0]
+        if not (
+            isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str)
+        ):
+            return self._slice_lines(source_lines, unit_start, unit_end), None
+
+        # Skip stripping when the docstring shares a line with the def/class (would
+        # leave broken syntax) or extends past the caller's slice (e.g. class_header).
+        ds_start = first.lineno
+        ds_end = first.end_lineno or first.lineno
+        if ds_start <= node.lineno or ds_end > unit_end:
+            return self._slice_lines(source_lines, unit_start, unit_end), None
+
+        before = source_lines[unit_start - 1 : ds_start - 1]
+        after = source_lines[ds_end:unit_end]
+        return "".join(before + after), docstring
+
+    def _emit_class_units(self, cls: ast.ClassDef, source_lines: list[str], cursor: int, units: list[_CodeUnit]) -> int:
+        """Emit class header and per-method units for ``cls``; return the next cursor (1-indexed)."""
+        class_start = cls.decorator_list[0].lineno if cls.decorator_list else cls.lineno
+        class_end = cls.end_lineno or cls.lineno
+        class_name = cls.name
+        class_decorators = [self._safe_unparse(d) for d in cls.decorator_list]
+
+        # Methods, async methods, and nested classes become their own units so a
+        # method is never split mid-statement.
+        split_children_idx = [
+            k
+            for k, child in enumerate(cls.body)
+            if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+        ]
+
+        # Bare class signature (decorators + ``class Foo(...):`` lines) used by
+        # ``preserve_class_definition`` to prefix later chunks of the same class.
+        class_signature: str | None = None
+        if cls.body:
+            body_start = cls.body[0].lineno
+            if body_start > class_start:
+                class_signature = self._slice_lines(source_lines, class_start, body_start - 1)
+
+        # Whole class fits in one unit when there are no inner split points.
+        if not split_children_idx:
+            unit_slice = self._slice_lines(source_lines, cursor, class_end)
+            stripped_docstring: str | None = None
+            if self.strip_docstrings:
+                unit_slice, stripped_docstring = self._strip_docstring(cls, source_lines, cursor, class_end)
+            units.append(
+                _CodeUnit(
+                    source=unit_slice,
+                    start_line=class_start,
+                    end_line=class_end,
+                    kind="class",
+                    name=class_name,
+                    class_name=class_name,
+                    class_signature=class_signature,
+                    decorators=class_decorators,
+                    docstring=stripped_docstring,
+                )
+            )
+            return class_end + 1
+
+        # Class header: from outer cursor up to (but excluding) the first split child.
+        first_child = cls.body[split_children_idx[0]]
+        if (
+            isinstance(first_child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+            and first_child.decorator_list
+        ):
+            first_child_start = first_child.decorator_list[0].lineno
+        else:
+            first_child_start = first_child.lineno
+        header_end = first_child_start - 1
+        header_slice = self._slice_lines(source_lines, cursor, header_end)
+        header_docstring: str | None = None
+        if self.strip_docstrings:
+            header_slice, header_docstring = self._strip_docstring(cls, source_lines, cursor, header_end)
+
+        units.append(
+            _CodeUnit(
+                source=header_slice,
+                start_line=class_start,
+                end_line=header_end,
+                kind="class_header",
+                name=class_name,
+                class_name=class_name,
+                class_signature=class_signature,
+                decorators=class_decorators,
+                docstring=header_docstring,
+            )
+        )
+        inner_cursor = header_end + 1
+
+        for idx in split_children_idx:
+            child = cls.body[idx]
+            if not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                continue  # narrowed above; kept for the type checker
+            child_start = child.decorator_list[0].lineno if child.decorator_list else child.lineno
+            child_end = child.end_lineno or child.lineno
+            decorators = [self._safe_unparse(d) for d in child.decorator_list]
+
+            unit_slice = self._slice_lines(source_lines, inner_cursor, child_end)
+            stripped_docstring = None
+            if self.strip_docstrings:
+                unit_slice, stripped_docstring = self._strip_docstring(child, source_lines, inner_cursor, child_end)
+
+            kind = "method" if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) else "nested_class"
+            units.append(
+                _CodeUnit(
+                    source=unit_slice,
+                    start_line=child_start,
+                    end_line=child_end,
+                    kind=kind,
+                    name=child.name,
+                    class_name=class_name,
+                    class_signature=class_signature,
+                    decorators=decorators,
+                    docstring=stripped_docstring,
+                )
+            )
+            inner_cursor = child_end + 1
+
+        # Append trailing class-body lines (comments / blanks after the last method).
+        if inner_cursor <= class_end and units:
+            trailing = self._slice_lines(source_lines, inner_cursor, class_end)
+            units[-1].source += trailing
+            units[-1].end_line = class_end
+
+        return class_end + 1
+
+    def _extract_units(self, source: str) -> list[_CodeUnit]:
+        """Parse ``source`` and produce the ordered list of syntactic split units."""
+        tree = ast.parse(source)
+        source_lines = source.splitlines(keepends=True)
+        total_lines = len(source_lines)
+
+        units: list[_CodeUnit] = []
+        cursor = 1
+        body = tree.body
+        node_idx = 0
+        node_count = len(body)
+
+        while node_idx < node_count:
+            node = body[node_idx]
+
+            # Module docstring (only valid as the very first statement).
+            if (
+                node_idx == 0
+                and isinstance(node, ast.Expr)
+                and isinstance(node.value, ast.Constant)
+                and isinstance(node.value.value, str)
+            ):
+                end = node.end_lineno or node.lineno
+                units.append(
+                    _CodeUnit(
+                        source=self._slice_lines(source_lines, cursor, end),
+                        start_line=node.lineno,
+                        end_line=end,
+                        kind="module_docstring",
+                    )
+                )
+                cursor = end + 1
+                node_idx += 1
+                continue
+
+            # Group consecutive imports into one unit.
+            if isinstance(node, (ast.Import, ast.ImportFrom)):
+                import_end_idx = node_idx
+                while import_end_idx < node_count and isinstance(body[import_end_idx], (ast.Import, ast.ImportFrom)):
+                    import_end_idx += 1
+                last = body[import_end_idx - 1]
+                end = last.end_lineno or last.lineno
+                units.append(
+                    _CodeUnit(
+                        source=self._slice_lines(source_lines, cursor, end),
+                        start_line=node.lineno,
+                        end_line=end,
+                        kind="imports",
+                    )
+                )
+                cursor = end + 1
+                node_idx = import_end_idx
+                continue
+
+            if isinstance(node, ast.ClassDef):
+                cursor = self._emit_class_units(node, source_lines, cursor, units)
+                node_idx += 1
+                continue
+
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                start = node.decorator_list[0].lineno if node.decorator_list else node.lineno
+                end = node.end_lineno or node.lineno
+                decorators = [self._safe_unparse(d) for d in node.decorator_list]
+                unit_slice = self._slice_lines(source_lines, cursor, end)
+                stripped_docstring: str | None = None
+                if self.strip_docstrings:
+                    unit_slice, stripped_docstring = self._strip_docstring(node, source_lines, cursor, end)
+                units.append(
+                    _CodeUnit(
+                        source=unit_slice,
+                        start_line=start,
+                        end_line=end,
+                        kind="function",
+                        name=node.name,
+                        decorators=decorators,
+                        docstring=stripped_docstring,
+                    )
+                )
+                cursor = end + 1
+                node_idx += 1
+                continue
+
+            # Catch-all for top-level statements (assignments, conditionals, etc.).
+            end = node.end_lineno or node.lineno
+            units.append(
+                _CodeUnit(
+                    source=self._slice_lines(source_lines, cursor, end),
+                    start_line=node.lineno,
+                    end_line=end,
+                    kind="statement",
+                )
+            )
+            cursor = end + 1
+            node_idx += 1
+
+        # Append trailing content (comments after the last node) so the split is loss-less.
+        if cursor <= total_lines and units:
+            trailing = self._slice_lines(source_lines, cursor, total_lines)
+            units[-1].source += trailing
+            units[-1].end_line = total_lines
+        elif cursor <= total_lines and not units:
+            units.append(
+                _CodeUnit(
+                    source=self._slice_lines(source_lines, cursor, total_lines),
+                    start_line=cursor,
+                    end_line=total_lines,
+                    kind="statement",
+                )
+            )
+
+        return units
+
+    def _merge_units(self, units: list[_CodeUnit]) -> list[list[_CodeUnit]]:
+        """Greedily merge units toward ``max_effective_lines``; oversized units become solo chunks."""
+        chunks: list[list[_CodeUnit]] = []
+        current: list[_CodeUnit] = []
+        current_lines = 0
+        target = self.max_effective_lines
+
+        def flush() -> None:
+            nonlocal current, current_lines
+            if current:
+                chunks.append(current)
+                current = []
+                current_lines = 0
+
+        for unit in units:
+            if self._is_oversized(unit):
+                flush()
+                chunks.append([unit])
+                continue
+
+            unit_eff = self._effective_lines(unit.source)
+
+            if not current:
+                current = [unit]
+                current_lines = unit_eff
+                continue
+
+            # Keep merging while below the minimum or while adding moves us closer to the target.
+            new_total = current_lines + unit_eff
+            if current_lines < self.min_effective_lines or abs(new_total - target) < abs(current_lines - target):
+                current.append(unit)
+                current_lines = new_total
+            else:
+                flush()
+                current = [unit]
+                current_lines = unit_eff
+
+        flush()
+        return chunks
+
+    @staticmethod
+    def _ordered_unique(items: list[str]) -> list[str]:
+        """Return the list of unique items in their first-seen order."""
+        return list(dict.fromkeys(items))
+
+    def _build_chunk_meta(self, chunk: list[_CodeUnit], parent_doc: Document) -> dict[str, Any]:
+        """Construct the output meta dict for a chunk of merged units."""
+        meta: dict[str, Any] = {}
+        if parent_doc.meta:
+            meta.update({k: v for k, v in parent_doc.meta.items() if k not in {"split_id"}})
+        meta["source_id"] = parent_doc.id
+
+        # Units are emitted in source order, so chunk[0]/chunk[-1] give the extremes.
+        meta["start_line"] = chunk[0].start_line
+        meta["end_line"] = chunk[-1].end_line
+        meta["unit_kinds"] = [u.kind for u in chunk]
+
+        include_classes = self._ordered_unique([u.class_name for u in chunk if u.class_name])
+        if include_classes:
+            meta["include_classes"] = include_classes
+
+        decorators: list[str] = []
+        for u in chunk:
+            decorators.extend(u.decorators)
+        decorators = self._ordered_unique(decorators)
+        if decorators:
+            meta["decorators"] = decorators
+
+        if self.strip_docstrings:
+            docstrings = [u.docstring for u in chunk if u.docstring]
+            if docstrings:
+                meta["docstrings"] = docstrings
+
+        return meta
+
+    def _render_chunk_content(self, chunk: list[_CodeUnit]) -> str:
+        """Render chunk content, optionally prefixing class signatures for orphan members."""
+        body = "".join(u.source for u in chunk)
+        if not self.preserve_class_definition:
+            return body
+
+        classes_with_header = {u.class_name for u in chunk if u.kind in {"class", "class_header"} and u.class_name}
+        prepended: list[str] = []
+        seen: set[str] = set()
+        for u in chunk:
+            if (
+                u.class_name
+                and u.class_name not in classes_with_header
+                and u.class_name not in seen
+                and u.class_signature
+            ):
+                prepended.append(u.class_signature)
+                seen.add(u.class_name)
+
+        if not prepended:
+            return body
+        return "".join(prepended) + body
+
+    def _secondary_split(self, unit: _CodeUnit, parent_doc: Document) -> list[Document]:
+        """Apply a line-based fallback split with overlap to a single oversized unit."""
+        qualified_name = unit.name or unit.kind
+        if unit.class_name and unit.name:
+            qualified_name = f"{unit.class_name}.{unit.name}"
+        logger.warning(
+            "Oversized {kind} '{func_name}' at lines {start}-{end} ({eff} effective lines) exceeds "
+            "{factor}x max_effective_lines={max_effective_lines}; falling back to line-based secondary split "
+            "with overlap={overlap}.",
+            kind=unit.kind,
+            func_name=qualified_name,
+            start=unit.start_line,
+            end=unit.end_line,
+            eff=self._effective_lines(unit.source),
+            factor=self.oversized_factor,
+            max_effective_lines=self.max_effective_lines,
+            overlap=self.secondary_split_overlap,
+        )
+
+        # DocumentSplitter measures in physical lines; this approximates effective lines.
+        split_length = (
+            self.secondary_split_length if self.secondary_split_length is not None else self.max_effective_lines
+        )
+        overlap = min(self.secondary_split_overlap, max(0, split_length - 1))
+
+        splitter = DocumentSplitter(split_by="line", split_length=split_length, split_overlap=overlap)
+        intermediate = splitter.run(documents=[Document(content=unit.source)])["documents"]
+
+        base_meta = self._build_chunk_meta([unit], parent_doc)
+        results: list[Document] = []
+        for idx, piece in enumerate(intermediate):
+            meta = dict(base_meta)
+            meta["secondary_split"] = True
+            meta["secondary_split_index"] = idx
+            meta["secondary_split_total"] = len(intermediate)
+            results.append(Document(content=piece.content or "", meta=meta))
+        return results
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Split each Python source ``Document`` into syntax-aware chunks.
+
+        :param documents: Documents whose ``content`` is Python source code. Each
+            document's ``meta`` is propagated onto its chunks.
+        :returns: ``{"documents": [...]}`` where each chunk's meta additionally carries
+            ``source_id``, ``split_id``, ``start_line``, ``end_line``, ``unit_kinds`` and
+            - where applicable - ``include_classes``, ``decorators``, ``docstrings``,
+            ``secondary_split``.
+        :raises ValueError: If any document's content is ``None``.
+        :raises TypeError: If any document's content is not a string.
+        :raises SyntaxError: If a document's content is not valid Python.
+        """
+        for doc in documents:
+            if doc.content is None:
+                raise ValueError(
+                    f"PythonCodeSplitter only works with text documents but content for document ID {doc.id} is None."
+                )
+            if not isinstance(doc.content, str):
+                raise TypeError("PythonCodeSplitter only works with text documents (str content).")
+
+        final_docs: list[Document] = []
+        for doc in documents:
+            assert doc.content is not None  # narrowed by the loop above
+            if not doc.content.strip():
+                logger.warning("Document ID {doc_id} has empty content. Skipping this document.", doc_id=doc.id)
+                continue
+
+            units = self._extract_units(doc.content)
+            if not units:
+                continue
+
+            chunks = self._merge_units(units)
+            split_id = 0
+            for chunk in chunks:
+                if len(chunk) == 1 and self._is_oversized(chunk[0]):
+                    for piece in self._secondary_split(chunk[0], doc):
+                        piece.meta["split_id"] = split_id
+                        split_id += 1
+                        final_docs.append(piece)
+                    continue
+
+                content = self._render_chunk_content(chunk)
+                meta = self._build_chunk_meta(chunk, doc)
+                meta["split_id"] = split_id
+                split_id += 1
+                final_docs.append(Document(content=content, meta=meta))
+
+        return {"documents": final_docs}

haystack/components/preprocessors/recursive_splitter.py

@@ -0,0 +1,484 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from copy import deepcopy
+from typing import Any, Literal
+
+from haystack import Document, component, logging
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install tiktoken'") as tiktoken_imports:
+    import tiktoken
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class RecursiveDocumentSplitter:
+    """
+    Recursively chunk text into smaller chunks.
+
+    This component is used to split text into smaller chunks, it does so by recursively applying a list of separators
+    to the text.
+
+    The separators are applied in the order they are provided, typically this is a list of separators that are
+    applied in a specific order, being the last separator the most specific one.
+
+    Each separator is applied to the text, it then checks each of the resulting chunks, it keeps the chunks that
+    are within the split_length, for the ones that are larger than the split_length, it applies the next separator in the
+    list to the remaining text.
+
+    This is done until all chunks are smaller than the split_length parameter.
+
+    Example:
+
+    ```python
+    from haystack import Document
+    from haystack.components.preprocessors import RecursiveDocumentSplitter
+
+    chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
+    text = ('''Artificial intelligence (AI) - Introduction
+
+    AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
+    AI technology is widely used throughout industry, government, and science. Some high-profile applications include advanced web search engines; recommendation systems; interacting via human speech; autonomous vehicles; generative and creative tools; and superhuman play and analysis in strategy games.''')
+    doc = Document(content=text)
+    doc_chunks = chunker.run([doc])
+    print(doc_chunks["documents"])
+    # [
+    # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []})
+    # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []})
+    # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []})
+    # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []})
+    # ]
+    ```
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        *,
+        split_length: int = 200,
+        split_overlap: int = 0,
+        split_unit: Literal["word", "char", "token"] = "word",
+        separators: list[str] | None = None,
+        sentence_splitter_params: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Initializes a RecursiveDocumentSplitter.
+
+        :param split_length: The maximum length of each chunk by default in words, but can be in characters or tokens.
+            See the `split_units` parameter.
+        :param split_overlap: The number of characters to overlap between consecutive chunks.
+        :param split_unit: The unit of the split_length parameter. It can be either "word", "char", or "token".
+            If "token" is selected, the text will be split into tokens using the tiktoken tokenizer (o200k_base).
+        :param separators: An optional list of separator strings to use for splitting the text. The string
+            separators will be treated as regular expressions unless the separator is "sentence", in that case the
+            text will be split into sentences using a custom sentence tokenizer based on NLTK.
+            See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter.
+            If no separators are provided, the default separators ["\\n\\n", "sentence", "\\n", " "] are used.
+        :param sentence_splitter_params: Optional parameters to pass to the sentence tokenizer.
+            See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter for more information.
+
+        :raises ValueError: If the overlap is greater than or equal to the chunk size or if the overlap is negative, or
+                            if any separator is not a string.
+        """
+        self.split_length = split_length
+        self.split_overlap = split_overlap
+        self.split_units = split_unit
+        self.separators = separators if separators else ["\n\n", "sentence", "\n", " "]  # default separators
+        self._check_params()
+        self.nltk_tokenizer = None
+        self.sentence_splitter_params = (
+            {"keep_white_spaces": True} if sentence_splitter_params is None else sentence_splitter_params
+        )
+        self.tiktoken_tokenizer: "tiktoken.Encoding" | None = None
+        self._is_warmed_up = False
+
+    def warm_up(self) -> None:
+        """
+        Warm up the sentence tokenizer and tiktoken tokenizer if needed.
+        """
+        if self._is_warmed_up:
+            return
+        if "sentence" in self.separators:
+            self.nltk_tokenizer = self._get_custom_sentence_tokenizer(self.sentence_splitter_params)
+        if self.split_units == "token":
+            tiktoken_imports.check()
+            self.tiktoken_tokenizer = tiktoken.get_encoding("o200k_base")
+        self._is_warmed_up = True
+
+    def _check_params(self) -> None:
+        if self.split_length < 1:
+            raise ValueError("Split length must be at least 1 character.")
+        if self.split_overlap < 0:
+            raise ValueError("Overlap must be greater than zero.")
+        if self.split_overlap >= self.split_length:
+            raise ValueError("Overlap cannot be greater than or equal to the chunk size.")
+        if not all(isinstance(separator, str) for separator in self.separators):
+            raise ValueError("All separators must be strings.")
+
+    @staticmethod
+    def _get_custom_sentence_tokenizer(sentence_splitter_params: dict[str, Any]) -> Any:
+        from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
+
+        return SentenceSplitter(**sentence_splitter_params)
+
+    def _split_chunk(self, current_chunk: str) -> tuple[str, str]:
+        """
+        Splits a chunk based on the split_length and split_units attribute.
+
+        :param current_chunk: The current chunk to be split.
+        :returns:
+            A tuple containing the current chunk and the remaining chunk.
+        """
+        if self.split_units == "word":
+            words = current_chunk.split()
+            current_chunk = " ".join(words[: self.split_length])
+            remaining_words = words[self.split_length :]
+            return current_chunk, " ".join(remaining_words)
+        if self.split_units == "char":
+            text = current_chunk
+            current_chunk = text[: self.split_length]
+            remaining_chars = text[self.split_length :]
+            return current_chunk, remaining_chars
+
+        # at this point we know that the tokenizer is already initialized
+        tokens = self.tiktoken_tokenizer.encode(current_chunk)  # type: ignore
+        current_tokens = tokens[: self.split_length]
+        remaining_tokens = tokens[self.split_length :]
+        return self.tiktoken_tokenizer.decode(current_tokens), self.tiktoken_tokenizer.decode(remaining_tokens)  # type: ignore
+
+    def _apply_overlap(self, chunks: list[str]) -> list[str]:
+        """
+        Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
+
+        Works for both word- and character-level splitting. It trims the last chunk if it exceeds the split_length and
+        adds the trimmed content to the next chunk. If the last chunk is still too long after trimming, it splits it
+        and adds the first chunk to the list. This process continues until the last chunk is within the split_length.
+
+        :param chunks: A list of text chunks.
+        :returns:
+            A list of text chunks with the overlap applied.
+        """
+        overlapped_chunks: list[str] = []
+
+        for idx, chunk in enumerate(chunks):
+            if idx == 0:
+                overlapped_chunks.append(chunk)
+                continue
+
+            # get the overlap between the current and previous chunk
+            overlap, prev_chunk = self._get_overlap(overlapped_chunks)
+            if overlap == prev_chunk:
+                logger.warning(
+                    "Overlap is the same as the previous chunk. "
+                    "Consider increasing the `split_length` parameter or decreasing the `split_overlap` parameter."
+                )
+
+            current_chunk = self._create_chunk_starting_with_overlap(chunk, overlap)
+
+            # if this new chunk exceeds 'split_length', trim it and move the remaining text to the next chunk
+            # if this is the last chunk, another new chunk will contain the trimmed text preceded by the overlap
+            # of the last chunk
+            if self._chunk_length(current_chunk) > self.split_length:
+                current_chunk, remaining_text = self._split_chunk(current_chunk)
+                if idx < len(chunks) - 1:
+                    if self.split_units == "word":
+                        chunks[idx + 1] = remaining_text + " " + chunks[idx + 1]
+                    elif self.split_units == "token":
+                        # For token-based splitting, combine at token level
+                        # at this point we know that the tokenizer is already initialized
+                        remaining_tokens = self.tiktoken_tokenizer.encode(remaining_text)  # type: ignore
+                        next_chunk_tokens = self.tiktoken_tokenizer.encode(chunks[idx + 1])  # type: ignore
+                        chunks[idx + 1] = self.tiktoken_tokenizer.decode(remaining_tokens + next_chunk_tokens)  # type: ignore
+                    else:  # char
+                        chunks[idx + 1] = remaining_text + chunks[idx + 1]
+                elif remaining_text:
+                    # create a new chunk with the trimmed text preceded by the overlap of the last chunk
+                    overlapped_chunks.append(current_chunk)
+                    chunk = remaining_text
+                    overlap, _ = self._get_overlap(overlapped_chunks)
+                    current_chunk = self._create_chunk_starting_with_overlap(chunk, overlap)
+
+            overlapped_chunks.append(current_chunk)
+
+            # it can still be that the new last chunk exceeds the 'split_length'
+            # continue splitting until the last chunk is within 'split_length'
+            if idx == len(chunks) - 1 and self._chunk_length(current_chunk) > self.split_length:
+                last_chunk = overlapped_chunks.pop()
+                first_chunk, remaining_chunk = self._split_chunk(last_chunk)
+                overlapped_chunks.append(first_chunk)
+
+                while remaining_chunk:
+                    # combine overlap with remaining chunk
+                    overlap, _ = self._get_overlap(overlapped_chunks)
+                    current = self._create_chunk_starting_with_overlap(remaining_chunk, overlap)
+
+                    # if it fits within split_length we are done
+                    if self._chunk_length(current) <= self.split_length:
+                        overlapped_chunks.append(current)
+                        break
+
+                    # otherwise split it again
+                    first_chunk, remaining_chunk = self._split_chunk(current)
+                    overlapped_chunks.append(first_chunk)
+
+        return overlapped_chunks
+
+    def _create_chunk_starting_with_overlap(self, chunk: str, overlap: str) -> str:
+        if self.split_units == "word":
+            current_chunk = overlap + " " + chunk
+        elif self.split_units == "token":
+            # For token-based splitting, combine at token level
+            # at this point we know that the tokenizer is already initialized
+            overlap_tokens = self.tiktoken_tokenizer.encode(overlap)  # type: ignore
+            chunk_tokens = self.tiktoken_tokenizer.encode(chunk)  # type: ignore
+            current_chunk = self.tiktoken_tokenizer.decode(overlap_tokens + chunk_tokens)  # type: ignore
+        else:  # char
+            current_chunk = overlap + chunk
+        return current_chunk
+
+    def _get_overlap(self, overlapped_chunks: list[str]) -> tuple[str, str]:
+        """Get the previous overlapped chunk instead of the original chunk."""
+        prev_chunk = overlapped_chunks[-1]
+        overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
+
+        if self.split_units == "word":
+            word_chunks = prev_chunk.split()
+            overlap = " ".join(word_chunks[overlap_start:])
+        elif self.split_units == "token":
+            # For token-based splitting, handle overlap at token level
+            # at this point we know that the tokenizer is already initialized
+            tokens = self.tiktoken_tokenizer.encode(prev_chunk)  # type: ignore
+            overlap_tokens = tokens[overlap_start:]
+            overlap = self.tiktoken_tokenizer.decode(overlap_tokens)  # type: ignore
+        else:  # char
+            overlap = prev_chunk[overlap_start:]
+
+        return overlap, prev_chunk
+
+    def _chunk_length(self, text: str) -> int:
+        """
+        Get the length of the chunk in the specified units (words, characters, or tokens).
+
+        :param text: The text to measure.
+        :returns: The length of the text in the specified units.
+        """
+        if self.split_units == "word":
+            words = [word for word in text.split(" ") if word]
+            return len(words)
+        if self.split_units == "char":
+            return len(text)
+        # token
+        # at this point we know that the tokenizer is already initialized
+        return len(self.tiktoken_tokenizer.encode(text))  # type: ignore
+
+    def _chunk_text(self, text: str) -> list[str]:
+        """
+        Recursive chunking algorithm that divides text into smaller chunks based on a list of separator characters.
+
+        It starts with a list of separator characters (e.g., ["\n\n", "sentence", "\n", " "]) and attempts to divide
+        the text using the first separator. If the resulting chunks are still larger than the specified chunk size,
+        it moves to the next separator in the list. This process continues recursively, progressively applying each
+        specific separator until the chunks meet the desired size criteria.
+
+        :param text: The text to be split into chunks.
+        :returns:
+            A list of text chunks.
+        """
+        if self._chunk_length(text) <= self.split_length:
+            return [text]
+
+        for curr_separator in self.separators:
+            if curr_separator == "sentence":
+                # re. ignore: correct SentenceSplitter initialization is checked at the initialization of the component
+                sentence_with_spans = self.nltk_tokenizer.split_sentences(text)  # type: ignore
+                splits = [sentence["sentence"] for sentence in sentence_with_spans]
+            else:
+                # add escape "\" to the separator and wrapped it in a group so that it's included in the splits as well
+                escaped_separator = re.escape(curr_separator)
+                escaped_separator = f"({escaped_separator})"
+
+                # split the text and merge every two consecutive splits, i.e.: the text and the separator after it
+                splits = re.split(escaped_separator, text)
+                splits = [
+                    "".join([splits[i], splits[i + 1]]) if i < len(splits) - 1 else splits[i]
+                    for i in range(0, len(splits), 2)
+                ]
+
+                # remove last split if it's empty
+                splits = splits[:-1] if splits[-1] == "" else splits
+
+            if len(splits) == 1:  # go to next separator, if current separator not found in the text
+                continue
+
+            chunks = []
+            current_chunk: list[str] = []
+            current_length = 0
+
+            # check splits, if any is too long, recursively chunk it, otherwise add to current chunk
+            for split in splits:
+                split_text = split
+
+                # if adding this split exceeds chunk_size, process current_chunk
+                if current_length + self._chunk_length(split_text) > self.split_length:
+                    # process current_chunk
+                    if current_chunk:  # keep the good splits
+                        chunks.append("".join(current_chunk))
+                        current_chunk = []
+                        current_length = 0
+
+                    # recursively handle splits that are too large
+                    if self._chunk_length(split_text) > self.split_length:
+                        if curr_separator == self.separators[-1]:
+                            # tried last separator, can't split further, do a fixed-split based on word/character/token
+                            fall_back_chunks = self._fall_back_to_fixed_chunking(split_text, self.split_units)
+                            chunks.extend(fall_back_chunks)
+                        else:
+                            chunks.extend(self._chunk_text(split_text))
+
+                    else:
+                        current_chunk.append(split_text)
+                        current_length += self._chunk_length(split_text)
+                else:
+                    current_chunk.append(split_text)
+                    current_length += self._chunk_length(split_text)
+
+            if current_chunk:
+                chunks.append("".join(current_chunk))
+
+            if self.split_overlap > 0:
+                chunks = self._apply_overlap(chunks)
+
+            if chunks:
+                return chunks
+
+        # if no separator worked, fall back to word- or character-level chunking
+        chunks = self._fall_back_to_fixed_chunking(text, self.split_units)
+        if self.split_overlap > 0:
+            chunks = self._apply_overlap(chunks)
+        return chunks
+
+    def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> list[str]:
+        """
+        Fall back to a fixed chunking approach if no separator works for the text.
+
+        Splits the text into smaller chunks based on the split_length and split_units attributes, either by words,
+        characters, or tokens.
+
+        :param text: The text to be split into chunks.
+        :param split_units: The unit of the split_length parameter. It can be either "word", "char", or "token".
+        :returns:
+            A list of text chunks.
+        """
+        chunks = []
+
+        if split_units == "word":
+            words = re.findall(r"\S+|\s+", text)
+            current_chunk = []
+            current_length = 0
+
+            for word in words:
+                if word != " ":
+                    current_chunk.append(word)
+                    current_length += 1
+                    if current_length == self.split_length and current_chunk:
+                        chunks.append("".join(current_chunk))
+                        current_chunk = []
+                        current_length = 0
+                else:
+                    current_chunk.append(word)
+
+            if current_chunk:
+                chunks.append("".join(current_chunk))
+        elif split_units == "char":
+            for i in range(0, self._chunk_length(text), self.split_length):
+                chunks.append(text[i : i + self.split_length])
+        else:  # token
+            # at this point we know that the tokenizer is already initialized
+            tokens = self.tiktoken_tokenizer.encode(text)  # type: ignore
+            for i in range(0, len(tokens), self.split_length):
+                chunk_tokens = tokens[i : i + self.split_length]
+                chunks.append(self.tiktoken_tokenizer.decode(chunk_tokens))  # type: ignore
+        return chunks
+
+    def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None:
+        prev_doc = new_docs[-1]
+        # curr_pos and split_idx_start are character offsets, so measure the
+        # overlap and range in characters too (not via _chunk_length, which returns a word/token count).
+        prev_doc_length = len(prev_doc.content)  # type: ignore
+        overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"])
+        if overlap_length > 0:
+            prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)})
+            new_doc.meta["_split_overlap"].append(
+                {"doc_id": prev_doc.id, "range": (prev_doc_length - overlap_length, prev_doc_length)}
+            )
+
+    def _run_one(self, doc: Document) -> list[Document]:
+        chunks = self._chunk_text(doc.content)  # type: ignore # the caller already check for a non-empty doc.content
+        chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks  # remove last empty chunk if it exists
+        current_position = 0
+        current_page = 1
+
+        new_docs: list[Document] = []
+
+        for split_nr, chunk in enumerate(chunks):
+            meta = deepcopy(doc.meta)
+            meta["parent_id"] = doc.id
+            meta["split_id"] = split_nr
+            meta["split_idx_start"] = current_position
+            meta["_split_overlap"] = [] if self.split_overlap > 0 else None
+            new_doc = Document(content=chunk, meta=meta)
+
+            # add overlap information to the previous and current doc
+            if split_nr > 0 and self.split_overlap > 0:
+                self._add_overlap_info(current_position, new_doc, new_docs)
+
+            # count page breaks in the chunk
+            current_page += chunk.count("\f")
+
+            # if there are consecutive page breaks at the end with no more text, adjust the page number
+            # e.g: "text\f\f\f" -> 3 page breaks, but current_page should be 1
+            consecutive_page_breaks = len(chunk) - len(chunk.rstrip("\f"))
+
+            if consecutive_page_breaks > 0:
+                new_doc.meta["page_number"] = current_page - consecutive_page_breaks
+            else:
+                new_doc.meta["page_number"] = current_page
+
+            # keep the new chunk doc and update the current position
+            new_docs.append(new_doc)
+            # Advance current_position by chunk length minus overlap.
+            # split_overlap is in split_units, not chars, so get the actual
+            # overlap string from _get_overlap() and use its char length.
+            if self.split_overlap > 0 and split_nr < len(chunks) - 1:
+                overlap_str, _ = self._get_overlap([doc.content for doc in new_docs])  # type: ignore[misc]
+                overlap_char_len = len(overlap_str)
+            else:
+                overlap_char_len = 0
+            current_position += len(chunk) - overlap_char_len
+
+        return new_docs
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Split a list of documents into documents with smaller chunks of text.
+
+        :param documents: List of Documents to split.
+        :returns:
+            A dictionary containing a key "documents" with a List of Documents with smaller chunks of text corresponding
+            to the input documents.
+        """
+        if not self._is_warmed_up and ("sentence" in self.separators or self.split_units == "token"):
+            self.warm_up()
+
+        docs = []
+        for doc in documents:
+            if not doc.content or doc.content == "":
+                logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
+                continue
+            docs.extend(self._run_one(doc))
+
+        return {"documents": docs}

haystack/components/preprocessors/sentence_tokenizer.py

@@ -0,0 +1,237 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from pathlib import Path
+from typing import Any, Literal
+
+from haystack import logging
+from haystack.lazy_imports import LazyImport
+
+with LazyImport("Run 'pip install nltk>=3.9.1'") as nltk_imports:
+    import nltk
+
+logger = logging.getLogger(__name__)
+
+Language = Literal[
+    "ru", "sl", "es", "sv", "tr", "cs", "da", "nl", "en", "et", "fi", "fr", "de", "el", "it", "no", "pl", "pt", "ml"
+]
+
+ISO639_TO_NLTK = {
+    "ru": "russian",
+    "sl": "slovene",
+    "es": "spanish",
+    "sv": "swedish",
+    "tr": "turkish",
+    "cs": "czech",
+    "da": "danish",
+    "nl": "dutch",
+    "en": "english",
+    "et": "estonian",
+    "fi": "finnish",
+    "fr": "french",
+    "de": "german",
+    "el": "greek",
+    "it": "italian",
+    "no": "norwegian",
+    "pl": "polish",
+    "pt": "portuguese",
+    "ml": "malayalam",
+}
+
+QUOTE_SPANS_RE = re.compile(r'"[^"]*"|\'[^\']*\'')
+
+if nltk_imports.is_successful():
+
+    def load_sentence_tokenizer(
+        language: Language, keep_white_spaces: bool = False
+    ) -> nltk.tokenize.punkt.PunktSentenceTokenizer:
+        """
+        Utility function to load the nltk sentence tokenizer.
+
+        :param language: The language for the tokenizer.
+        :param keep_white_spaces: If True, the tokenizer will keep white spaces between sentences.
+        :returns: nltk sentence tokenizer.
+        """
+        try:
+            nltk.data.find("tokenizers/punkt_tab")
+        except LookupError:
+            try:
+                nltk.download("punkt_tab")
+            except FileExistsError as error:
+                logger.debug("NLTK punkt tokenizer seems to be already downloaded. Error message: {error}", error=error)
+
+        language_name = ISO639_TO_NLTK.get(language)
+
+        if language_name is not None:
+            sentence_tokenizer = nltk.data.load(f"tokenizers/punkt_tab/{language_name}.pickle")
+        else:
+            logger.warning(
+                "PreProcessor couldn't find the default sentence tokenizer model for {language}. "
+                " Using English instead. You may train your own model and use the 'tokenizer_model_folder' parameter.",
+                language=language,
+            )
+            sentence_tokenizer = nltk.data.load("tokenizers/punkt_tab/english.pickle")
+
+        if keep_white_spaces:
+            sentence_tokenizer._lang_vars = CustomPunktLanguageVars()
+
+        return sentence_tokenizer
+
+    class CustomPunktLanguageVars(nltk.tokenize.punkt.PunktLanguageVars):
+        # The following adjustment of PunktSentenceTokenizer is inspired by:
+        # https://stackoverflow.com/questions/33139531/preserve-empty-lines-with-nltks-punkt-tokenizer
+        # It is needed for preserving whitespace while splitting text into sentences.
+        _period_context_fmt = r"""
+                %(SentEndChars)s             # a potential sentence ending
+                \s*                          # match potential whitespace [ \t\n\x0B\f\r]
+                (?=(?P<after_tok>
+                    %(NonWord)s              # either other punctuation
+                    |
+                    (?P<next_tok>\S+)        # or some other token - original version: \s+(?P<next_tok>\S+)
+                ))"""
+
+        def period_context_re(self) -> re.Pattern:
+            """
+            Compiles and returns a regular expression to find contexts including possible sentence boundaries.
+
+            :returns: A compiled regular expression pattern.
+            """
+            try:
+                return self._re_period_context  # type: ignore
+            except:  # noqa: E722
+                self._re_period_context = re.compile(
+                    self._period_context_fmt
+                    % {
+                        "NonWord": self._re_non_word_chars,
+                        # SentEndChars might be followed by closing brackets, so we match them here.
+                        "SentEndChars": self._re_sent_end_chars + r"[\)\]}]*",
+                    },
+                    re.UNICODE | re.VERBOSE,
+                )
+                return self._re_period_context
+
+
+class SentenceSplitter:
+    """
+    SentenceSplitter splits a text into sentences using the nltk sentence tokenizer
+    """
+
+    def __init__(
+        self,
+        language: Language = "en",
+        use_split_rules: bool = True,
+        extend_abbreviations: bool = True,
+        keep_white_spaces: bool = False,
+    ) -> None:
+        """
+        Initializes the SentenceSplitter with the specified language, split rules, and abbreviation handling.
+
+        :param language: The language for the tokenizer. Default is "en".
+        :param use_split_rules: If True, the additional split rules are used. If False, the rules are not used.
+        :param extend_abbreviations: If True, the abbreviations used by NLTK's PunktTokenizer are extended by a list
+            of curated abbreviations if available. If False, the default abbreviations are used.
+            Currently supported languages are: en, de.
+        :param keep_white_spaces: If True, the tokenizer will keep white spaces between sentences.
+        """
+        nltk_imports.check()
+        self.language = language
+        # after checking nltk_imports, we are sure that load_sentence_tokenizer is defined
+        self.sentence_tokenizer = load_sentence_tokenizer(language, keep_white_spaces=keep_white_spaces)
+        self.use_split_rules = use_split_rules
+        if extend_abbreviations:
+            abbreviations = SentenceSplitter._read_abbreviations(language)
+            self.sentence_tokenizer._params.abbrev_types.update(abbreviations)
+        self.keep_white_spaces = keep_white_spaces
+
+    def split_sentences(self, text: str) -> list[dict[str, Any]]:
+        """
+        Splits a text into sentences including references to original char positions for each split.
+
+        :param text: The text to split.
+        :returns: list of sentences with positions.
+        """
+        sentence_spans = list(self.sentence_tokenizer.span_tokenize(text))
+        if self.use_split_rules:
+            sentence_spans = SentenceSplitter._apply_split_rules(text, sentence_spans)
+
+        return [{"sentence": text[start:end], "start": start, "end": end} for start, end in sentence_spans]
+
+    @staticmethod
+    def _apply_split_rules(text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
+        """
+        Applies additional split rules to the sentence spans.
+
+        :param text: The text to split.
+        :param sentence_spans: The list of sentence spans to split.
+        :returns: The list of sentence spans after applying the split rules.
+        """
+        new_sentence_spans = []
+        quote_spans = [match.span() for match in QUOTE_SPANS_RE.finditer(text)]
+        while sentence_spans:
+            span = sentence_spans.pop(0)
+            next_span = sentence_spans[0] if len(sentence_spans) > 0 else None
+            while next_span and SentenceSplitter._needs_join(text, span, next_span, quote_spans):
+                sentence_spans.pop(0)
+                span = (span[0], next_span[1])
+                next_span = sentence_spans[0] if len(sentence_spans) > 0 else None
+            start, end = span
+            new_sentence_spans.append((start, end))
+        return new_sentence_spans
+
+    @staticmethod
+    def _needs_join(
+        text: str, span: tuple[int, int], next_span: tuple[int, int], quote_spans: list[tuple[int, int]]
+    ) -> bool:
+        """
+        Checks if the spans need to be joined as parts of one sentence.
+
+        This method determines whether two adjacent sentence spans should be joined back together as a single sentence.
+        It's used to prevent incorrect sentence splitting in specific cases like quotations, numbered lists,
+        and parenthetical expressions.
+
+        :param text: The text containing the spans.
+        :param span: Tuple of (start, end) positions for the current sentence span.
+        :param next_span: Tuple of (start, end) positions for the next sentence span.
+        :param quote_spans: All quoted spans within text.
+        :returns:
+            True if the spans needs to be joined.
+        """
+        start, end = span
+        next_start, next_end = next_span
+
+        # sentence. sentence"\nsentence -> no split (end << quote_end)
+        # sentence.", sentence -> no split (end < quote_end)
+        # sentence?", sentence -> no split (end < quote_end)
+        if any(quote_start < end < quote_end for quote_start, quote_end in quote_spans):
+            # sentence boundary is inside a quote
+            return True
+
+        # sentence." sentence -> split (end == quote_end)
+        # sentence?" sentence -> no split (end == quote_end)
+        if any(quote_start < end == quote_end and text[quote_end - 2] == "?" for quote_start, quote_end in quote_spans):
+            # question is cited
+            return True
+
+        if re.search(r"(^|\n)\s*\d{1,2}\.$", text[start:end]) is not None:
+            # sentence ends with a numeration
+            return True
+
+        # next sentence starts with a bracket or we return False
+        return re.search(r"^\s*[\(\[]", text[next_start:next_end]) is not None
+
+    @staticmethod
+    def _read_abbreviations(lang: Language) -> list[str]:
+        """
+        Reads the abbreviations for a given language from the abbreviations file.
+
+        :param lang: The language to read the abbreviations for.
+        :returns: List of abbreviations.
+        """
+        abbreviations_file = Path(__file__).parent.parent.parent / f"data/abbreviations/{lang}.txt"
+        if not abbreviations_file.exists():
+            logger.warning("No abbreviations file found for {language}. Using default abbreviations.", language=lang)
+            return []
+
+        return abbreviations_file.read_text().split("\n")

haystack/components/preprocessors/text_cleaner.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+import string
+from typing import Any
+
+from haystack import component
+
+
+@component
+class TextCleaner:
+    """
+    Cleans text strings.
+
+    It can remove substrings matching a list of regular expressions, convert text to lowercase,
+    remove punctuation, and remove numbers.
+    Use it to clean up text data before evaluation.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.preprocessors import TextCleaner
+
+    text_to_clean = "1Moonlight shimmered softly, 300 Wolves howled nearby, Night enveloped everything."
+
+    cleaner = TextCleaner(convert_to_lowercase=True, remove_punctuation=False, remove_numbers=True)
+    result = cleaner.run(texts=[text_to_clean])
+    ```
+    """
+
+    def __init__(
+        self,
+        remove_regexps: list[str] | None = None,
+        convert_to_lowercase: bool = False,
+        remove_punctuation: bool = False,
+        remove_numbers: bool = False,
+    ) -> None:
+        """
+        Initializes the TextCleaner component.
+
+        :param remove_regexps: A list of regex patterns to remove matching substrings from the text.
+        :param convert_to_lowercase: If `True`, converts all characters to lowercase.
+        :param remove_punctuation: If `True`, removes punctuation from the text.
+        :param remove_numbers: If `True`, removes numerical digits from the text.
+        """
+        self._remove_regexps = remove_regexps
+        self._convert_to_lowercase = convert_to_lowercase
+        self._remove_punctuation = remove_punctuation
+        self._remove_numbers = remove_numbers
+
+        self._regex = None
+        if remove_regexps:
+            self._regex = re.compile("|".join(remove_regexps), flags=re.IGNORECASE)
+        to_remove = ""
+        if remove_punctuation:
+            to_remove = string.punctuation
+        if remove_numbers:
+            to_remove += string.digits
+
+        self._translator = str.maketrans("", "", to_remove) if to_remove else None
+
+    @component.output_types(texts=list[str])
+    def run(self, texts: list[str]) -> dict[str, Any]:
+        """
+        Cleans up the given list of strings.
+
+        :param texts: List of strings to clean.
+        :returns: A dictionary with the following key:
+            - `texts`:  the cleaned list of strings.
+        """
+
+        if self._regex:
+            texts = [self._regex.sub("", text) for text in texts]
+
+        if self._convert_to_lowercase:
+            texts = [text.lower() for text in texts]
+
+        if self._translator:
+            texts = [text.translate(self._translator) for text in texts]
+
+        return {"texts": texts}

haystack/components/query/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"query_expander": ["QueryExpander"]}
+
+if TYPE_CHECKING:
+    from .query_expander import QueryExpander as QueryExpander
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/query/query_expander.py

@@ -0,0 +1,373 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import default_from_dict, default_to_dict, logging
+from haystack.components.builders.prompt_builder import PromptBuilder
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.component import component
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_PROMPT_TEMPLATE = """
+You are part of an information system that processes user queries for retrieval.
+You have to expand a given query into {{ n_expansions }} queries that are
+semantically similar to improve retrieval recall.
+
+Structure:
+Follow the structure shown below in examples to generate expanded queries.
+
+Examples:
+1.  Query: "climate change effects"
+    {"queries": ["impact of climate change", "consequences of global warming", "effects of environmental changes"]}
+
+2.  Query: "machine learning algorithms"
+    {"queries": ["neural networks", "clustering techniques", "supervised learning methods", "deep learning models"]}
+
+3.  Query: "open source NLP frameworks"
+    {"queries": ["natural language processing tools", "free nlp libraries", "open-source NLP platforms"]}
+
+Guidelines:
+- Generate queries that use different words and phrasings
+- Include synonyms and related terms
+- Maintain the same core meaning and intent
+- Make queries that are likely to retrieve relevant information the original might miss
+- Focus on variations that would work well with keyword-based search
+- Respond in the same language as the input query
+
+Your Task:
+Query: "{{ query }}"
+
+You *must* respond with a JSON object containing a "queries" array with the expanded queries.
+Example: {"queries": ["query1", "query2", "query3"]}"""
+
+
+@component
+class QueryExpander:
+    """
+    A component that returns a list of semantically similar queries to improve retrieval recall in RAG systems.
+
+    The component uses a chat generator to expand queries. The chat generator is expected to return a JSON response
+    with the following structure:
+    ```json
+    {"queries": ["expanded query 1", "expanded query 2", "expanded query 3"]}
+    ```
+
+    ### Usage example
+
+    ```python
+    from haystack.components.generators.chat.openai import OpenAIChatGenerator
+    from haystack.components.query import QueryExpander
+
+    expander = QueryExpander(
+        chat_generator=OpenAIChatGenerator(model="gpt-4.1-mini"),
+        n_expansions=3
+    )
+
+    result = expander.run(query="green energy sources")
+    print(result["queries"])
+    # Output: ['alternative query 1', 'alternative query 2', 'alternative query 3', 'green energy sources']
+    # Note: Up to 3 additional queries + 1 original query (if include_original_query=True)
+
+    # To control total number of queries:
+    expander = QueryExpander(n_expansions=2, include_original_query=True)  # Up to 3 total
+    # or
+    expander = QueryExpander(n_expansions=3, include_original_query=False)  # Exactly 3 total
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator | None = None,
+        prompt_template: str | None = None,
+        n_expansions: int = 4,
+        include_original_query: bool = True,
+    ) -> None:
+        """
+        Initialize the QueryExpander component.
+
+        :param chat_generator: The chat generator component to use for query expansion.
+            If None, a default OpenAIChatGenerator with gpt-4.1-mini model is used.
+        :param prompt_template: Custom [PromptBuilder](https://docs.haystack.deepset.ai/docs/promptbuilder)
+            template for query expansion. The template should instruct the LLM to return a JSON response with the
+            structure: `{"queries": ["query1", "query2", "query3"]}`. The template should include 'query' and
+            'n_expansions' variables.
+        :param n_expansions: Number of alternative queries to generate (default: 4).
+        :param include_original_query: Whether to include the original query in the output.
+        """
+        if n_expansions <= 0:
+            raise ValueError("n_expansions must be positive")
+
+        self.n_expansions = n_expansions
+        self.include_original_query = include_original_query
+
+        if chat_generator is None:
+            self.chat_generator: ChatGenerator = OpenAIChatGenerator(
+                model="gpt-4.1-mini",
+                generation_kwargs={
+                    "temperature": 0.7,
+                    "response_format": {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": "query_expansion",
+                            "schema": {
+                                "type": "object",
+                                "properties": {"queries": {"type": "array", "items": {"type": "string"}}},
+                                "required": ["queries"],
+                                "additionalProperties": False,
+                            },
+                        },
+                    },
+                    "seed": 42,
+                },
+            )
+        else:
+            self.chat_generator = chat_generator
+
+        self.prompt_template = prompt_template or DEFAULT_PROMPT_TEMPLATE
+
+        # Check if required variables are present in the template
+        if "query" not in self.prompt_template:
+            logger.warning(
+                "The prompt template does not contain the 'query' variable. This may cause issues during execution."
+            )
+        if "n_expansions" not in self.prompt_template:
+            logger.warning(
+                "The prompt template does not contain the 'n_expansions' variable. "
+                "This may cause issues during execution."
+            )
+
+        self._prompt_builder = PromptBuilder(
+            template=self.prompt_template, required_variables=["n_expansions", "query"]
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :return: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(self.chat_generator, name="chat_generator"),
+            prompt_template=self.prompt_template,
+            n_expansions=self.n_expansions,
+            include_original_query=self.include_original_query,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "QueryExpander":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data: Dictionary with serialized data.
+        :return: Deserialized component.
+        """
+        init_params = data.get("init_parameters", {})
+
+        deserialize_chatgenerator_inplace(init_params, key="chat_generator")
+
+        return default_from_dict(cls, data)
+
+    @component.output_types(queries=list[str])
+    def run(self, query: str, n_expansions: int | None = None) -> dict[str, list[str]]:
+        """
+        Expand the input query into multiple semantically similar queries.
+
+        The language of the original query is preserved in the expanded queries.
+
+        :param query: The original query to expand.
+        :param n_expansions: Number of additional queries to generate (not including the original).
+            If None, uses the value from initialization. Must be a positive integer.
+        :return: Dictionary with "queries" key containing the list of expanded queries.
+            If include_original_query=True, the original query will be included in addition
+            to the n_expansions alternative queries.
+        :raises ValueError: If n_expansions is not positive (less than or equal to 0).
+        """
+
+        self.warm_up()
+
+        response = {"queries": [query] if self.include_original_query else []}
+
+        if not query.strip():
+            logger.warning("Empty query provided to QueryExpander")
+            return response
+
+        expansion_count = n_expansions if n_expansions is not None else self.n_expansions
+        if expansion_count <= 0:
+            raise ValueError("n_expansions must be positive")
+
+        try:
+            prompt_result = self._prompt_builder.run(query=query.strip(), n_expansions=expansion_count)
+            generator_result = self.chat_generator.run(messages=[ChatMessage.from_user(prompt_result["prompt"])])
+
+            if not generator_result.get("replies") or len(generator_result["replies"]) == 0:
+                logger.warning("ChatGenerator returned no replies for query: {query}", query=query)
+                return response
+
+            expanded_text = generator_result["replies"][0].text.strip()
+            expanded_queries = self._parse_expanded_queries(expanded_text)
+
+            # Limit the number of expanded queries to the requested amount
+            if len(expanded_queries) > expansion_count:
+                logger.warning(
+                    "Generated {generated_count} queries but only {requested_count} were requested. "
+                    "Truncating to the first {requested_count} queries. ",
+                    generated_count=len(expanded_queries),
+                    requested_count=expansion_count,
+                )
+                expanded_queries = expanded_queries[:expansion_count]
+
+            # Add original query if requested and remove duplicates
+            if self.include_original_query:
+                expanded_queries_lower = [q.lower() for q in expanded_queries]
+                if query.lower() not in expanded_queries_lower:
+                    expanded_queries.append(query)
+
+            response["queries"] = expanded_queries
+            return response
+
+        except Exception as e:
+            # Fallback: return original query to maintain pipeline functionality
+            logger.exception("Failed to expand query {query}: {error}", query=query, error=str(e))
+            return response
+
+    @component.output_types(queries=list[str])
+    async def run_async(self, query: str, n_expansions: int | None = None) -> dict[str, list[str]]:
+        """
+        Asynchronously expand the input query into multiple semantically similar queries.
+
+        The language of the original query is preserved in the expanded queries.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in an async code. If the chat generator only implements a synchronous
+        `run` method, it is executed in a thread to avoid blocking the event loop.
+
+        :param query: The original query to expand.
+        :param n_expansions: Number of additional queries to generate (not including the original).
+            If None, uses the value from initialization. Must be a positive integer.
+        :return: Dictionary with "queries" key containing the list of expanded queries.
+            If include_original_query=True, the original query will be included in addition
+            to the n_expansions alternative queries.
+        :raises ValueError: If n_expansions is not positive (less than or equal to 0).
+        """
+
+        await self.warm_up_async()
+
+        response = {"queries": [query] if self.include_original_query else []}
+
+        if not query.strip():
+            logger.warning("Empty query provided to QueryExpander")
+            return response
+
+        expansion_count = n_expansions if n_expansions is not None else self.n_expansions
+        if expansion_count <= 0:
+            raise ValueError("n_expansions must be positive")
+
+        try:
+            prompt_result = self._prompt_builder.run(query=query.strip(), n_expansions=expansion_count)
+            generator_result = await _execute_component_async(
+                self.chat_generator, messages=[ChatMessage.from_user(prompt_result["prompt"])]
+            )
+
+            if not generator_result.get("replies") or len(generator_result["replies"]) == 0:
+                logger.warning("ChatGenerator returned no replies for query: {query}", query=query)
+                return response
+
+            expanded_text = generator_result["replies"][0].text.strip()
+            expanded_queries = self._parse_expanded_queries(expanded_text)
+
+            # Limit the number of expanded queries to the requested amount
+            if len(expanded_queries) > expansion_count:
+                logger.warning(
+                    "Generated {generated_count} queries but only {requested_count} were requested. "
+                    "Truncating to the first {requested_count} queries. ",
+                    generated_count=len(expanded_queries),
+                    requested_count=expansion_count,
+                )
+                expanded_queries = expanded_queries[:expansion_count]
+
+            # Add original query if requested and remove duplicates
+            if self.include_original_query:
+                expanded_queries_lower = [q.lower() for q in expanded_queries]
+                if query.lower() not in expanded_queries_lower:
+                    expanded_queries.append(query)
+
+            response["queries"] = expanded_queries
+            return response
+
+        except Exception as e:
+            # Fallback: return original query to maintain pipeline functionality
+            logger.exception("Failed to expand query {query}: {error}", query=query, error=str(e))
+            return response
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator.
+        """
+        if hasattr(self.chat_generator, "warm_up"):
+            self.chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator on the serving event loop.
+        """
+        if hasattr(self.chat_generator, "warm_up_async"):
+            await self.chat_generator.warm_up_async()
+        elif hasattr(self.chat_generator, "warm_up"):
+            self.chat_generator.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's resources.
+        """
+        if hasattr(self.chat_generator, "close"):
+            self.chat_generator.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's async resources.
+        """
+        if hasattr(self.chat_generator, "close_async"):
+            await self.chat_generator.close_async()
+        elif hasattr(self.chat_generator, "close"):
+            self.chat_generator.close()
+
+    @staticmethod
+    def _parse_expanded_queries(generator_response: str) -> list[str]:
+        """
+        Parse the generator response to extract individual expanded queries.
+
+        :param generator_response: The raw text response from the generator.
+        :return: List of parsed expanded queries.
+        """
+        parsed = _parse_dict_from_json(generator_response, expected_keys=["queries"], raise_on_failure=False)
+
+        if parsed is None:
+            return []
+
+        if not isinstance(parsed["queries"], list):
+            logger.warning(
+                "Expected 'queries' to be a list but got {type}. Returning no expanded queries.",
+                type=type(parsed["queries"]).__name__,
+            )
+            return []
+
+        queries = []
+        for item in parsed["queries"]:
+            if isinstance(item, str) and item.strip():
+                queries.append(item.strip())
+            else:
+                logger.warning("Skipping non-string or empty query in response: {item}", item=item)
+
+        return queries

haystack/components/rankers/__init__.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "llm_ranker": ["LLMRanker"],
+    "lost_in_the_middle": ["LostInTheMiddleRanker"],
+    "meta_field": ["MetaFieldRanker"],
+    "meta_field_grouping_ranker": ["MetaFieldGroupingRanker"],
+}
+
+if TYPE_CHECKING:
+    from .llm_ranker import LLMRanker as LLMRanker
+    from .lost_in_the_middle import LostInTheMiddleRanker as LostInTheMiddleRanker
+    from .meta_field import MetaFieldRanker as MetaFieldRanker
+    from .meta_field_grouping_ranker import MetaFieldGroupingRanker as MetaFieldGroupingRanker
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/rankers/llm_ranker.py

@@ -0,0 +1,403 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _deduplicate_documents, _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+def _default_openai_chat_generator() -> ChatGenerator:
+    return OpenAIChatGenerator(
+        model="gpt-4.1-mini",
+        generation_kwargs={
+            "temperature": 0.0,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "document_ranking",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "documents": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {"index": {"type": "integer"}},
+                                    "required": ["index"],
+                                    "additionalProperties": False,
+                                },
+                            }
+                        },
+                        "required": ["documents"],
+                        "additionalProperties": False,
+                    },
+                },
+            },
+        },
+    )
+
+
+DEFAULT_PROMPT_TEMPLATE = """
+You are ranking retrieved documents for relevance to a query.
+
+Return valid JSON only, with this structure:
+{
+  "documents": [
+    {"index": 1}
+  ]
+}
+
+Rules:
+- Rank documents from most relevant to least relevant for answering the query.
+- Only include documents that are relevant to the query.
+- Do not return or rank documents that are not relevant.
+- If none are relevant, return {"documents": []}.
+- Use only document indices from the provided documents.
+- Do not repeat document indices.
+- Do not include explanations or any text outside the JSON object.
+
+Query:
+{{ query }}
+
+Documents:
+{% for document in documents %}
+Document {{ loop.index }}:
+content: {{ document.content or "" }}
+
+{% endfor %}
+""".strip()
+
+
+@component
+class LLMRanker:
+    """
+    Ranks documents for a query using a Large Language Model.
+
+    The LLM is expected to return a JSON object containing ranked document indices.
+
+    Usage example:
+
+    ```python
+    from haystack import Document
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.rankers import LLMRanker
+
+    chat_generator = OpenAIChatGenerator(
+        model="gpt-4.1-mini",
+        generation_kwargs={
+            "temperature": 0.0,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "document_ranking",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "documents": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {"index": {"type": "integer"}},
+                                    "required": ["index"],
+                                    "additionalProperties": False,
+                                },
+                            }
+                        },
+                        "required": ["documents"],
+                        "additionalProperties": False,
+                    },
+                },
+            },
+        },
+    )
+
+    ranker = LLMRanker(chat_generator=chat_generator)
+
+    documents = [
+        Document(id="paris", content="Paris is the capital of France."),
+        Document(id="berlin", content="Berlin is the capital of Germany."),
+    ]
+
+    result = ranker.run(query="capital of Germany", documents=documents)
+    print(result["documents"][0].id)
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator | None = None,
+        prompt: str = DEFAULT_PROMPT_TEMPLATE,
+        top_k: int = 10,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        Initialize the LLMRanker component.
+
+        :param chat_generator:
+            The chat generator to use for reranking. If `None`, a default `OpenAIChatGenerator` configured for JSON
+            output is used.
+        :param prompt:
+            Custom prompt template for reranking. The prompt must include exactly the variables `query` and
+            `documents` and instruct the LLM to return ranked 1-based document indices as JSON.
+        :param top_k:
+            The maximum number of documents to return.
+        :param raise_on_failure:
+            If `True`, raise when generation or response parsing fails. If `False`, log the failure and return the
+            input documents in fallback order.
+        """
+        if top_k <= 0:
+            raise ValueError(f"top_k must be > 0, but got {top_k}")
+
+        self.top_k = top_k
+        self.raise_on_failure = raise_on_failure
+        self.prompt = prompt
+        self._prompt_builder = PromptBuilder(template=self.prompt, required_variables=["documents", "query"])
+        if set(self._prompt_builder.variables) != {"documents", "query"}:
+            raise ValueError("prompt must include exactly the variables 'documents' and 'query'.")
+
+        if chat_generator is None:
+            self._chat_generator = _default_openai_chat_generator()
+        else:
+            self._chat_generator = chat_generator
+
+    def warm_up(self) -> None:
+        """Warm up the underlying chat generator."""
+        if hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """Warm up the underlying chat generator on the serving event loop."""
+        if hasattr(self._chat_generator, "warm_up_async"):
+            await self._chat_generator.warm_up_async()
+        elif hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    def close(self) -> None:
+        """Release the underlying chat generator's resources."""
+        if hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    async def close_async(self) -> None:
+        """Release the underlying chat generator's async resources."""
+        if hasattr(self._chat_generator, "close_async"):
+            await self._chat_generator.close_async()
+        elif hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            prompt=self.prompt,
+            top_k=self.top_k,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMRanker":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of the component.
+        :returns:
+            The deserialized component instance.
+        """
+        init_params = data.get("init_parameters", {})
+        if init_params.get("chat_generator"):
+            deserialize_chatgenerator_inplace(init_params, key="chat_generator")
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document])
+    def run(self, query: str, documents: list[Document], top_k: int | None = None) -> dict[str, list[Document]]:
+        """
+        Rank documents for a query using an LLM.
+
+        Before ranking, duplicate documents are removed.
+
+        :param query:
+            The query used for reranking.
+        :param documents:
+            Candidate documents to rerank.
+        :param top_k:
+            The maximum number of documents to return. Overrides the instance's `top_k` if provided.
+        :returns:
+            A dictionary with the ranked documents under the `documents` key.
+        """
+        if top_k is not None and top_k <= 0:
+            raise ValueError(f"top_k must be > 0, but got {top_k}")
+
+        if not documents:
+            return {"documents": []}
+
+        top_k = self.top_k if top_k is None else top_k
+        deduplicated_documents = _deduplicate_documents(documents)
+        fallback_documents = deduplicated_documents
+
+        if not query.strip():
+            logger.warning("Empty query provided to LLMRanker. Returning documents without reranking.")
+            return {"documents": fallback_documents}
+
+        self.warm_up()
+
+        prompt = self._prompt_builder.run(query=query.strip(), documents=deduplicated_documents)
+
+        try:
+            result = self._chat_generator.run(messages=[ChatMessage.from_user(prompt["prompt"])])
+        except Exception as exc:
+            if self.raise_on_failure:
+                raise
+            logger.warning(
+                "LLMRanker failed during chat generation. Returning fallback order. Error: {error}", error=exc
+            )
+            return {"documents": fallback_documents}
+
+        try:
+            reply_text = self._get_reply_text(result)
+            ranked_documents = self._rank_documents_from_reply(reply_text=reply_text, documents=deduplicated_documents)
+        except (TypeError, ValueError) as exc:
+            if self.raise_on_failure:
+                raise
+            logger.warning(
+                "LLMRanker failed while processing the chat response. Returning fallback order. Error: {error}",
+                error=exc,
+            )
+            return {"documents": fallback_documents}
+
+        return {"documents": ranked_documents[:top_k]}
+
+    @component.output_types(documents=list[Document])
+    async def run_async(
+        self, query: str, documents: list[Document], top_k: int | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Asynchronously rank documents for a query using an LLM.
+
+        Before ranking, duplicate documents are removed.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in an async code. If the chat generator only implements a synchronous
+        `run` method, it is executed in a thread to avoid blocking the event loop.
+
+        :param query:
+            The query used for reranking.
+        :param documents:
+            Candidate documents to rerank.
+        :param top_k:
+            The maximum number of documents to return. Overrides the instance's `top_k` if provided.
+        :returns:
+            A dictionary with the ranked documents under the `documents` key.
+        """
+        if top_k is not None and top_k <= 0:
+            raise ValueError(f"top_k must be > 0, but got {top_k}")
+
+        if not documents:
+            return {"documents": []}
+
+        top_k = self.top_k if top_k is None else top_k
+        deduplicated_documents = _deduplicate_documents(documents)
+        fallback_documents = deduplicated_documents
+
+        if not query.strip():
+            logger.warning("Empty query provided to LLMRanker. Returning documents without reranking.")
+            return {"documents": fallback_documents}
+
+        await self.warm_up_async()
+
+        prompt = self._prompt_builder.run(query=query.strip(), documents=deduplicated_documents)
+
+        try:
+            result = await _execute_component_async(
+                self._chat_generator, messages=[ChatMessage.from_user(prompt["prompt"])]
+            )
+        except Exception as exc:
+            if self.raise_on_failure:
+                raise
+            logger.warning(
+                "LLMRanker failed during chat generation. Returning fallback order. Error: {error}", error=exc
+            )
+            return {"documents": fallback_documents}
+
+        try:
+            reply_text = self._get_reply_text(result)
+            ranked_documents = self._rank_documents_from_reply(reply_text=reply_text, documents=deduplicated_documents)
+        except (TypeError, ValueError) as exc:
+            if self.raise_on_failure:
+                raise
+            logger.warning(
+                "LLMRanker failed while processing the chat response. Returning fallback order. Error: {error}",
+                error=exc,
+            )
+            return {"documents": fallback_documents}
+
+        return {"documents": ranked_documents[:top_k]}
+
+    @staticmethod
+    def _get_reply_text(result: dict[str, Any]) -> str:
+        replies = result.get("replies") or []
+        if not replies:
+            raise ValueError("ChatGenerator returned no replies.")
+
+        reply_text = replies[0].text
+        if reply_text is None:
+            raise ValueError("ChatGenerator returned a reply without text.")
+
+        return reply_text
+
+    @staticmethod
+    def _rank_documents_from_reply(reply_text: str, documents: list[Document]) -> list[Document]:
+        parsed_response = _parse_dict_from_json(reply_text, expected_keys=["documents"], raise_on_failure=True)
+        ranked_entries = parsed_response["documents"]
+
+        if not isinstance(ranked_entries, list):
+            raise TypeError("Expected 'documents' in ranking response to be a list.")
+
+        if not ranked_entries:
+            return []
+
+        ranked_documents: list[Document] = []
+
+        for entry in ranked_entries:
+            if not isinstance(entry, dict):
+                raise TypeError("Expected each ranked document entry to be a JSON object.")
+
+            document_index = entry.get("index")
+            if document_index is None:
+                continue
+
+            try:
+                # LLMs can return numeric indices as strings even when asked for integers.
+                document_index = int(document_index)
+            except (TypeError, ValueError):
+                continue
+
+            # Jinja's `loop.index` is 1-based:
+            # https://jinja.palletsprojects.com/en/stable/templates/#for
+            if document_index < 1 or document_index > len(documents):
+                continue
+
+            document = documents[document_index - 1]
+            ranked_documents.append(document)
+
+        if not ranked_documents:
+            raise ValueError("Ranking response did not contain any valid document indices.")
+
+        return ranked_documents

haystack/components/rankers/lost_in_the_middle.py

@@ -0,0 +1,137 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+from haystack import Document, component
+from haystack.utils.misc import _deduplicate_documents
+
+
+@component
+class LostInTheMiddleRanker:
+    """
+    A LostInTheMiddle Ranker.
+
+    Ranks documents based on the 'lost in the middle' order so that the most relevant documents are either at the
+    beginning or end, while the least relevant are in the middle.
+
+    LostInTheMiddleRanker assumes that some prior component in the pipeline has already ranked documents by relevance
+    and requires no query as input but only documents. It is typically used as the last component before building a
+    prompt for an LLM to prepare the input context for the LLM.
+
+    Lost in the Middle ranking lays out document contents into LLM context so that the most relevant contents are at
+    the beginning or end of the input context, while the least relevant is in the middle of the context. See the
+    paper ["Lost in the Middle: How Language Models Use Long Contexts"](https://arxiv.org/abs/2307.03172) for more
+    details.
+
+    Usage example:
+    ```python
+    from haystack.components.rankers import LostInTheMiddleRanker
+    from haystack import Document
+
+    ranker = LostInTheMiddleRanker()
+    docs = [Document(content="Paris"), Document(content="Berlin"), Document(content="Madrid")]
+    result = ranker.run(documents=docs)
+    for doc in result["documents"]:
+        print(doc.content)
+    ```
+    """
+
+    def __init__(self, word_count_threshold: int | None = None, top_k: int | None = None) -> None:
+        """
+        Initialize the LostInTheMiddleRanker.
+
+        If 'word_count_threshold' is specified, this ranker includes all documents up until the point where adding
+        another document would exceed the 'word_count_threshold'. The last document that causes the threshold to
+        be breached will be included in the resulting list of documents, but all subsequent documents will be
+        discarded.
+
+        :param word_count_threshold: The maximum total number of words across all documents selected by the ranker.
+        :param top_k: The maximum number of documents to return.
+        """
+        if isinstance(word_count_threshold, int) and word_count_threshold <= 0:
+            raise ValueError(
+                f"Invalid value for word_count_threshold: {word_count_threshold}. word_count_threshold must be > 0."
+            )
+        if isinstance(top_k, int) and top_k <= 0:
+            raise ValueError(f"top_k must be > 0, but got {top_k}")
+
+        self.word_count_threshold = word_count_threshold
+        self.top_k = top_k
+
+    @component.output_types(documents=list[Document])
+    def run(
+        self, documents: list[Document], top_k: int | None = None, word_count_threshold: int | None = None
+    ) -> dict[str, list[Document]]:
+        """
+        Reranks documents based on the "lost in the middle" order.
+
+        Before ranking, documents are deduplicated by their id, retaining only the document with the highest score
+        if a score is present.
+
+        :param documents: List of Documents to reorder.
+        :param top_k: The maximum number of documents to return.
+        :param word_count_threshold: The maximum total number of words across all documents selected by the ranker.
+        :returns:
+            A dictionary with the following keys:
+            - `documents`: Reranked list of Documents
+
+        :raises ValueError:
+            If any of the documents is not textual.
+        """
+        if isinstance(word_count_threshold, int) and word_count_threshold <= 0:
+            raise ValueError(
+                f"Invalid value for word_count_threshold: {word_count_threshold}. word_count_threshold must be > 0."
+            )
+        if isinstance(top_k, int) and top_k <= 0:
+            raise ValueError(f"top_k must be > 0, but got {top_k}")
+
+        if not documents:
+            return {"documents": []}
+
+        top_k = top_k or self.top_k
+        word_count_threshold = word_count_threshold or self.word_count_threshold
+
+        deduplicated_documents = _deduplicate_documents(documents)
+        documents_to_reorder = deduplicated_documents[:top_k] if top_k else deduplicated_documents
+
+        # If there's only one document, return it as is
+        if len(documents_to_reorder) == 1:
+            return {"documents": documents_to_reorder}
+
+        # Raise an error if any document is not textual
+        if any(not doc.content_type == "text" for doc in documents_to_reorder):
+            raise ValueError("Some provided documents are not textual; LostInTheMiddleRanker can process only text.")
+
+        # Initialize word count and indices for the "lost in the middle" order
+        word_count = 0
+        document_index = list(range(len(documents_to_reorder)))
+        lost_in_the_middle_indices = [0]
+
+        # If word count threshold is set and the first document has content, calculate word count for the first document
+        if word_count_threshold and documents_to_reorder[0].content:
+            word_count = len(documents_to_reorder[0].content.split())
+
+            # If the first document already meets the word count threshold, return it
+            if word_count >= word_count_threshold:
+                return {"documents": [documents_to_reorder[0]]}
+
+        # Start from the second document and create "lost in the middle" order
+        for doc_idx in document_index[1:]:
+            # Calculate the index at which the current document should be inserted
+            insertion_index = len(lost_in_the_middle_indices) // 2 + len(lost_in_the_middle_indices) % 2
+
+            # Insert the document index at the calculated position
+            lost_in_the_middle_indices.insert(insertion_index, doc_idx)
+
+            # If word count threshold is set and the document has content, calculate the total word count
+            if word_count_threshold and documents_to_reorder[doc_idx].content:
+                word_count += len(documents_to_reorder[doc_idx].content.split())  # type: ignore[union-attr]
+
+                # If the total word count meets the threshold, stop processing further documents
+                if word_count >= word_count_threshold:
+                    break
+
+        # Documents in the "lost in the middle" order
+        ranked_docs = [documents_to_reorder[idx] for idx in lost_in_the_middle_indices]
+        return {"documents": ranked_docs}