chore: import upstream snapshot with attribution

haystack/utils/__init__.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "auth": ["Secret", "deserialize_secrets_inplace"],
+    "azure": ["default_azure_ad_token_provider"],
+    "base_serialization": ["_deserialize_value_with_schema", "_serialize_value_with_schema"],
+    "callable_serialization": ["deserialize_callable", "serialize_callable"],
+    "device": ["ComponentDevice", "Device", "DeviceMap", "DeviceType"],
+    "deserialization": ["deserialize_chatgenerator_inplace", "deserialize_component_inplace"],
+    "filters": ["document_matches_filter"],
+    "jinja2_extensions": ["Jinja2TimeExtension"],
+    "jupyter": ["is_in_jupyter"],
+    "misc": ["expit", "expand_page_range"],
+    "requests_utils": ["request_with_retry", "async_request_with_retry"],
+    "type_serialization": ["deserialize_type", "serialize_type"],
+}
+
+if TYPE_CHECKING:
+    from .auth import Secret as Secret
+    from .auth import deserialize_secrets_inplace as deserialize_secrets_inplace
+    from .azure import default_azure_ad_token_provider as default_azure_ad_token_provider
+    from .base_serialization import _deserialize_value_with_schema as _deserialize_value_with_schema
+    from .base_serialization import _serialize_value_with_schema as _serialize_value_with_schema
+    from .callable_serialization import deserialize_callable as deserialize_callable
+    from .callable_serialization import serialize_callable as serialize_callable
+    from .deserialization import deserialize_chatgenerator_inplace as deserialize_chatgenerator_inplace
+    from .deserialization import deserialize_component_inplace as deserialize_component_inplace
+    from .device import ComponentDevice as ComponentDevice
+    from .device import Device as Device
+    from .device import DeviceMap as DeviceMap
+    from .device import DeviceType as DeviceType
+    from .filters import document_matches_filter as document_matches_filter
+    from .jinja2_extensions import Jinja2TimeExtension as Jinja2TimeExtension
+    from .jupyter import is_in_jupyter as is_in_jupyter
+    from .misc import expand_page_range as expand_page_range
+    from .misc import expit as expit
+    from .requests_utils import async_request_with_retry as async_request_with_retry
+    from .requests_utils import request_with_retry as request_with_retry
+    from .type_serialization import deserialize_type as deserialize_type
+    from .type_serialization import serialize_type as serialize_type
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/utils/async_utils.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+from typing import Any
+
+from haystack import logging
+
+logger = logging.getLogger(__name__)
+
+
+async def _execute_component_async(component_instance: Any, **kwargs: Any) -> dict[str, Any]:
+    """
+    Run a component asynchronously, preferring its `run_async` method when implemented.
+
+    If the component does not implement `run_async`, its synchronous `run` method is executed in a thread
+    to avoid blocking the event loop.
+
+    :param component_instance: The component to run. Any object exposing a `run` method and optionally a
+        `run_async` coroutine method.
+    :param kwargs: Keyword arguments passed to the component's `run_async` or `run` method.
+    :returns:
+        The component's output dictionary.
+    """
+    run_async = getattr(component_instance, "run_async", None)
+    if callable(run_async):
+        return await run_async(**kwargs)
+
+    logger.debug(
+        "{component_type} does not implement 'run_async'. Running the synchronous 'run' method in a thread "
+        "to avoid blocking the event loop.",
+        component_type=type(component_instance).__name__,
+    )
+    return await asyncio.to_thread(component_instance.run, **kwargs)

haystack/utils/auth.py

@@ -0,0 +1,238 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class SecretType(Enum):
+    """
+    Type of secret: token (API key) or environment variable.
+    """
+
+    TOKEN = "token"
+    ENV_VAR = "env_var"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "SecretType":
+        """
+        Convert a string to a SecretType.
+
+        :param string: The string to convert.
+        """
+        mapping = {e.value: e for e in SecretType}
+        _type = mapping.get(string)
+        if _type is None:
+            raise ValueError(f"Unknown secret type '{string}'")
+        return _type
+
+
+class Secret(ABC):
+    """
+    Encapsulates a secret used for authentication.
+
+    Usage example:
+    ```python
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.utils import Secret
+
+    generator = OpenAIChatGenerator(api_key=Secret.from_token("<here_goes_your_token>"))
+    ```
+    """
+
+    @staticmethod
+    def from_token(token: str) -> "Secret":
+        """
+        Create a token-based secret. Cannot be serialized.
+
+        :param token:
+            The token to use for authentication.
+        """
+        return TokenSecret(_token=token)
+
+    @staticmethod
+    def from_env_var(env_vars: str | list[str], *, strict: bool = True) -> "Secret":
+        """
+        Create an environment variable-based secret. Accepts one or more environment variables.
+
+        Upon resolution, it returns a string token from the first environment variable that is set.
+
+        :param env_vars:
+            A single environment variable or an ordered list of
+            candidate environment variables.
+        :param strict:
+            Whether to raise an exception if none of the environment
+            variables are set.
+        """
+        if isinstance(env_vars, str):
+            env_vars = [env_vars]
+        return EnvVarSecret(_env_vars=tuple(env_vars), _strict=strict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Convert the secret to a JSON-serializable dictionary.
+
+        Some secrets may not be serializable.
+
+        :returns:
+            The serialized policy.
+        """
+        out = {"type": self.type.value}
+        inner = self._to_dict()
+        assert all(k not in inner for k in out)
+        out.update(inner)
+        return out
+
+    @staticmethod
+    def from_dict(dict: dict[str, Any]) -> "Secret":  # noqa:A002
+        """
+        Create a secret from a JSON-serializable dictionary.
+
+        :param dict:
+            The dictionary with the serialized data.
+        :returns:
+            The deserialized secret.
+        """
+        secret_map = {SecretType.TOKEN: TokenSecret, SecretType.ENV_VAR: EnvVarSecret}
+        secret_type = SecretType.from_str(dict["type"])
+        return secret_map[secret_type]._from_dict(dict)  # type: ignore
+
+    @abstractmethod
+    def resolve_value(self) -> Any | None:
+        """
+        Resolve the secret to an atomic value. The semantics of the value is secret-dependent.
+
+        :returns:
+            The value of the secret, if any.
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def type(self) -> SecretType:
+        """
+        The type of the secret.
+        """
+        pass
+
+    @abstractmethod
+    def _to_dict(self) -> dict[str, Any]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def _from_dict(_: dict[str, Any]) -> "Secret":
+        pass
+
+
+@dataclass(frozen=True)
+class TokenSecret(Secret):
+    """
+    A secret that uses a string token/API key.
+
+    Cannot be serialized.
+    """
+
+    _token: str
+    _type: SecretType = SecretType.TOKEN
+
+    def __post_init__(self) -> None:
+        super().__init__()
+        assert self._type == SecretType.TOKEN
+
+        if len(self._token) == 0:
+            raise ValueError("Authentication token cannot be empty.")
+
+    def _to_dict(self) -> dict[str, Any]:
+        raise ValueError(
+            "Cannot serialize token-based secret. Use an alternative secret type like environment variables."
+        )
+
+    @staticmethod
+    def _from_dict(_: dict[str, Any]) -> "Secret":
+        raise ValueError(
+            "Cannot deserialize token-based secret. Use an alternative secret type like environment variables."
+        )
+
+    def __repr__(self) -> str:
+        # Hide the token so it can't leak through print/log/traceback formatting.
+        return f"TokenSecret(_token=<redacted>, _type={self._type!r})"
+
+    def resolve_value(self) -> Any | None:
+        """Return the token."""
+        return self._token
+
+    @property
+    def type(self) -> SecretType:
+        """The type of the secret."""
+        return self._type
+
+
+@dataclass(frozen=True)
+class EnvVarSecret(Secret):
+    """
+    A secret that accepts one or more environment variables.
+
+    Upon resolution, it returns a string token from the first environment variable that is set. Can be serialized.
+    """
+
+    _env_vars: tuple[str, ...]
+    _strict: bool = True
+    _type: SecretType = SecretType.ENV_VAR
+
+    def __post_init__(self) -> None:
+        super().__init__()
+        assert self._type == SecretType.ENV_VAR
+
+        if len(self._env_vars) == 0:
+            raise ValueError("One or more environment variables must be provided for the secret.")
+
+    def _to_dict(self) -> dict[str, Any]:
+        return {"env_vars": list(self._env_vars), "strict": self._strict}
+
+    @staticmethod
+    def _from_dict(dictionary: dict[str, Any]) -> "Secret":
+        return EnvVarSecret(tuple(dictionary["env_vars"]), _strict=dictionary["strict"])
+
+    def resolve_value(self) -> Any | None:
+        """Resolve the secret to an atomic value. The semantics of the value is secret-dependent."""
+        out = None
+        for env_var in self._env_vars:
+            value = os.getenv(env_var)
+            if value is not None:
+                out = value
+                break
+        if out is None and self._strict:
+            raise ValueError(f"None of the following authentication environment variables are set: {self._env_vars}")
+        return out
+
+    @property
+    def type(self) -> SecretType:
+        """The type of the secret."""
+        return self._type
+
+
+def deserialize_secrets_inplace(data: dict[str, Any], keys: Iterable[str], *, recursive: bool = False) -> None:
+    """
+    Deserialize secrets in a dictionary inplace.
+
+    :param data:
+        The dictionary with the serialized data.
+    :param keys:
+        The keys of the secrets to deserialize.
+    :param recursive:
+        Whether to recursively deserialize nested dictionaries.
+    """
+    for k, v in data.items():
+        if isinstance(v, dict) and recursive:
+            deserialize_secrets_inplace(v, keys)
+        elif k in keys and v is not None:
+            data[k] = Secret.from_dict(v)

haystack/utils/azure.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(message="Run 'pip install azure-identity") as azure_import:
+    from azure.identity import DefaultAzureCredential, get_bearer_token_provider
+
+
+def default_azure_ad_token_provider() -> str:
+    """
+    Get a Azure AD token using the DefaultAzureCredential and the "https://cognitiveservices.azure.com/.default" scope.
+    """
+    azure_import.check()
+    return get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")()

haystack/utils/base_serialization.py

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from typing import Any
+
+import pydantic
+
+from haystack import logging
+from haystack.core.errors import DeserializationError
+from haystack.core.serialization import generate_qualified_class_name, import_class_by_name
+from haystack.utils import deserialize_callable, serialize_callable
+
+logger = logging.getLogger(__name__)
+
+_PRIMITIVE_TO_SCHEMA_MAP = {type(None): "null", bool: "boolean", int: "integer", float: "number", str: "string"}
+
+
+def _serialize_value_with_schema(payload: Any) -> dict[str, Any]:  # noqa: PLR0911
+    """
+    Serializes a value into a schema-aware format suitable for storage or transmission.
+
+    The output format separates the schema information from the actual data, making it easier
+    to deserialize complex nested structures correctly.
+
+    The function handles:
+    - Objects with to_dict() methods (e.g. dataclasses)
+    - Objects with __dict__ attributes
+    - Dictionaries
+    - Lists, tuples, and sets. Lists with mixed types are not supported.
+    - Primitive types (str, int, float, bool, None)
+
+    This is for runtime values (Agent/State data, pipeline inputs/outputs at a breakpoint), not
+    Component definitions — see `default_to_dict` in `core/serialization.py` for that other format.
+    Don't merge the two; they're not interchangeable.
+
+    :param payload: The value to serialize (can be any type)
+    :returns: The serialized dict representation of the given value. Contains two keys:
+        - "serialization_schema": Contains type information for each field.
+        - "serialized_data": Contains the actual data in a simplified format.
+
+    """
+    # Handle pydantic
+    if isinstance(payload, pydantic.BaseModel):
+        type_name = generate_qualified_class_name(type(payload))
+        return {"serialization_schema": {"type": type_name}, "serialized_data": payload.model_dump()}
+
+    # Handle dictionary case - iterate through fields
+    if isinstance(payload, dict):
+        schema: dict[str, Any] = {}
+        data: dict[str, Any] = {}
+
+        for field, val in payload.items():
+            # Recursively serialize each field
+            serialized_value = _serialize_value_with_schema(val)
+            schema[field] = serialized_value["serialization_schema"]
+            data[field] = serialized_value["serialized_data"]
+
+        return {"serialization_schema": {"type": "object", "properties": schema}, "serialized_data": data}
+
+    # Handle array case - iterate through elements
+    if isinstance(payload, (list, tuple, set)):
+        # Serialize each item in the array
+        serialized_list = []
+        for item in payload:
+            serialized_value = _serialize_value_with_schema(item)
+            serialized_list.append(serialized_value["serialized_data"])
+
+        # Determine item type from first element (if any)
+        # NOTE: We do not support mixed-type lists
+        if payload:
+            first = next(iter(payload))
+            item_schema = _serialize_value_with_schema(first)
+            base_schema = {"type": "array", "items": item_schema["serialization_schema"]}
+        else:
+            base_schema = {"type": "array", "items": {}}
+
+        # Add JSON Schema properties to infer sets and tuples
+        if isinstance(payload, set):
+            base_schema["uniqueItems"] = True
+        elif isinstance(payload, tuple):
+            base_schema["minItems"] = len(payload)
+            base_schema["maxItems"] = len(payload)
+
+        return {"serialization_schema": base_schema, "serialized_data": serialized_list}
+
+    # Handle Haystack style objects (e.g. dataclasses and Components)
+    if hasattr(payload, "to_dict") and callable(payload.to_dict):
+        type_name = generate_qualified_class_name(type(payload))
+        schema = {"type": type_name}
+        return {"serialization_schema": schema, "serialized_data": payload.to_dict()}
+
+    # Handle callable functions serialization
+    if callable(payload) and not isinstance(payload, type):
+        serialized = serialize_callable(payload)
+        return {"serialization_schema": {"type": "typing.Callable"}, "serialized_data": serialized}
+
+    # Handle Enums
+    if isinstance(payload, Enum):
+        type_name = generate_qualified_class_name(type(payload))
+        return {"serialization_schema": {"type": type_name}, "serialized_data": payload.name}
+
+    # Handle arbitrary objects with __dict__
+    if hasattr(payload, "__dict__"):
+        type_name = generate_qualified_class_name(type(payload))
+        schema = {"type": type_name}
+        serialized_data = {}
+        for key, value in vars(payload).items():
+            serialized_value = _serialize_value_with_schema(value)
+            serialized_data[key] = serialized_value["serialized_data"]
+        return {"serialization_schema": schema, "serialized_data": serialized_data}
+
+    # Handle primitives
+    schema = {"type": _primitive_schema_type(payload)}
+    return {"serialization_schema": schema, "serialized_data": payload}
+
+
+def _primitive_schema_type(value: Any) -> str:
+    """
+    Helper function to determine the schema type for primitive values.
+    """
+    for py_type, schema_value in _PRIMITIVE_TO_SCHEMA_MAP.items():
+        if isinstance(value, py_type):
+            return schema_value
+    logger.warning(
+        "Unsupported primitive type '{value_type}', falling back to 'string'", value_type=type(value).__name__
+    )
+    return "string"  # fallback
+
+
+def _deserialize_value_with_schema(serialized: dict[str, Any]) -> Any:
+    """
+    Deserializes a value with schema information back to its original form.
+
+    Takes a dict of the form:
+      {
+         "serialization_schema": {"type": "integer"} or {"type": "object", "properties": {...}},
+         "serialized_data": <the actual data>
+      }
+
+    NOTE: For array types we only support homogeneous lists (all elements of the same type).
+
+    :param serialized: The serialized dict with schema and data.
+    :returns: The deserialized value in its original form.
+    """
+
+    if not serialized or "serialization_schema" not in serialized or "serialized_data" not in serialized:
+        raise DeserializationError(
+            f"Invalid format of passed serialized payload. Expected a dictionary with keys "
+            f"'serialization_schema' and 'serialized_data'. Got: {serialized}"
+        )
+    schema = serialized["serialization_schema"]
+    data = serialized["serialized_data"]
+
+    schema_type = schema.get("type")
+
+    if not schema_type:
+        # for backward compatibility till Haystack 2.16 we use legacy implementation
+        raise DeserializationError(
+            "Missing 'type' key in 'serialization_schema'. This likely indicates that you're using a serialized "
+            "State object created with a version of Haystack older than 2.15.0. "
+            "Support for the old serialization format is removed in Haystack 2.16.0. "
+            "Please upgrade to the new serialization format to ensure forward compatibility."
+        )
+
+    # Handle object case (dictionary with properties)
+    if schema_type == "object":
+        properties = schema["properties"]
+        result: dict[str, Any] = {}
+        for field, raw_value in data.items():
+            field_schema = properties[field]
+            # Recursively deserialize each field - avoid creating temporary dict
+            result[field] = _deserialize_value_with_schema(
+                {"serialization_schema": field_schema, "serialized_data": raw_value}
+            )
+        return result
+
+    # Handle array case
+    if schema_type == "array":
+        # Deserialize each item
+        deserialized_items = [
+            _deserialize_value_with_schema({"serialization_schema": schema["items"], "serialized_data": item})
+            for item in data
+        ]
+        final_array: list | set | tuple
+        # Is a set if uniqueItems is True
+        if schema.get("uniqueItems") is True:
+            final_array = set(deserialized_items)
+        # Is a tuple if minItems and maxItems are set
+        elif schema.get("minItems") is not None and schema.get("maxItems") is not None:
+            final_array = tuple(deserialized_items)
+        else:
+            # Otherwise, it's a list
+            final_array = list(deserialized_items)
+        return final_array
+
+    # Handle primitive types
+    if schema_type in _PRIMITIVE_TO_SCHEMA_MAP.values():
+        return data
+
+    # Handle callable functions
+    if schema_type == "typing.Callable":
+        return deserialize_callable(data)
+
+    # Handle custom class types
+    return _deserialize_value({"type": schema_type, "data": data})
+
+
+def _deserialize_value(value: dict[str, Any]) -> Any:
+    """
+    Helper function to deserialize values from their envelope format {"type": T, "data": D}.
+
+    This handles:
+    - Custom classes (with a from_dict method)
+    - Enums
+    - Fallback for arbitrary classes (sets attributes on a blank instance)
+
+    :param value: The value to deserialize
+    :returns:
+        The deserialized value
+    :raises DeserializationError:
+        If the type cannot be imported or the value is not valid for the type.
+    """
+    # 1) Envelope case
+    value_type = value["type"]
+    payload = value["data"]
+
+    # Custom class where value_type is a qualified class name
+    # ValueError covers type names without a module prefix, which import_class_by_name cannot split
+    try:
+        cls = import_class_by_name(value_type)
+    except (ImportError, ValueError) as e:
+        raise DeserializationError(f"Class '{value_type}' not correctly imported") from e
+
+    # try from_dict (e.g. Haystack dataclasses and Components)
+    if hasattr(cls, "from_dict") and callable(cls.from_dict):
+        return cls.from_dict(payload)
+
+    # handle pydantic models
+    if issubclass(cls, pydantic.BaseModel):
+        try:
+            return cls.model_validate(payload)
+        except Exception as e:
+            raise DeserializationError(
+                f"Failed to deserialize data '{payload}' into Pydantic model '{value_type}'"
+            ) from e
+
+    # handle enum types
+    if issubclass(cls, Enum):
+        try:
+            return cls[payload]
+        except Exception as e:
+            raise DeserializationError(f"Value '{payload}' is not a valid member of Enum '{value_type}'") from e
+
+    # fallback: set attributes on a blank instance
+    deserialized_payload = {k: _deserialize_value(v) for k, v in payload.items()}
+    instance = cls.__new__(cls)
+    for attr_name, attr_value in deserialized_payload.items():
+        setattr(instance, attr_name, attr_value)
+    return instance

haystack/utils/callable_serialization.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import inspect
+from collections.abc import Callable
+from typing import Any
+
+from haystack import logging
+from haystack.core.errors import DeserializationError, SerializationError
+from haystack.core.serialization_security import (
+    _check_module_allowed,
+    _check_not_denied_builtin,
+    _is_denied_builtin,
+    _is_module_allowed,
+)
+from haystack.utils.type_serialization import thread_safe_import
+
+logger = logging.getLogger(__name__)
+
+
+def serialize_callable(callable_handle: Callable) -> str:
+    """
+    Serializes a callable to its full path.
+
+    :param callable_handle: The callable to serialize
+    :return: The full path of the callable
+    """
+    try:
+        full_arg_spec = inspect.getfullargspec(callable_handle)
+        is_instance_method = bool(full_arg_spec.args and full_arg_spec.args[0] == "self")
+    except TypeError:
+        is_instance_method = False
+    if is_instance_method:
+        raise SerializationError("Serialization of instance methods is not supported.")
+
+    # __qualname__ contains the fully qualified path we need for classmethods and staticmethods
+    qualname = getattr(callable_handle, "__qualname__", "")
+    if "<lambda>" in qualname:
+        raise SerializationError("Serialization of lambdas is not supported.")
+    if "<locals>" in qualname:
+        raise SerializationError("Serialization of nested functions is not supported.")
+
+    name = qualname or callable_handle.__name__
+
+    # Get the full package path of the function
+    module = inspect.getmodule(callable_handle)
+    if module is not None:
+        full_path = f"{module.__name__}.{name}"
+    else:
+        full_path = name
+
+    # Serialization succeeds, but a denied builtin (e.g. `eval`) won't reload without `unsafe=True`.
+    if _is_denied_builtin(callable_handle):
+        logger.warning(
+            "Serialized callable '{full_path}' is a builtin that is blocked during deserialization; "
+            "the resulting pipeline will only be loadable with unsafe=True.",
+            full_path=full_path,
+        )
+
+    return full_path
+
+
+def deserialize_callable(callable_handle: str) -> Callable:
+    """
+    Deserializes a callable given its full import path as a string.
+
+    Every module path tried during resolution is checked against the
+    deserialization allowlist (see `haystack.core.serialization_security`). Callables in modules
+    outside the allowlist are rejected with a `DeserializationError` before any import is
+    attempted. To allow a third-party module, extend the allowlist via
+    `Pipeline.load(..., allowed_modules=[...])`, `allow_deserialization_module(...)`, or the
+    `HAYSTACK_DESERIALIZATION_ALLOWLIST` environment variable.
+
+    :param callable_handle: The full path of the callable_handle
+    :return: The callable
+    :raises DeserializationError:
+        If the module path is not on the deserialization allowlist, or if the callable cannot
+        be found.
+    """
+    # Import here to avoid circular imports
+    from haystack.hooks.from_function import FunctionHook
+    from haystack.tools.tool import Tool
+
+    parts = callable_handle.split(".")
+
+    # Allow if any prefix is on the allowlist; checking each one individually would wrongly
+    # reject patterns like `j*on` against `json.dumps` (matches `json`, not the full handle).
+    if not any(_is_module_allowed(".".join(parts[:i])) for i in range(1, len(parts) + 1)):
+        _check_module_allowed(callable_handle)  # raises with the standard help message
+
+    for i in range(len(parts), 0, -1):
+        module_name = ".".join(parts[:i])
+        try:
+            mod: Any = thread_safe_import(module_name)
+        except Exception:
+            # keep reducing i until we find a valid module import
+            continue
+
+        attr_value = mod
+        for part in parts[i:]:
+            try:
+                attr_value = getattr(attr_value, part)
+            except AttributeError as e:
+                container = getattr(attr_value, "__name__", type(attr_value).__name__)
+                raise DeserializationError(f"Could not find attribute '{part}' in {container}") from e
+
+        # when the attribute is a classmethod, we need the underlying function
+        if isinstance(attr_value, (classmethod, staticmethod)):
+            attr_value = attr_value.__func__
+
+        # Handle the case where @tool decorator replaced the function with a Tool object
+        if isinstance(attr_value, Tool):
+            attr_value = attr_value.function or attr_value.async_function
+
+        # Handle the case where @hook decorator replaced the function with a FunctionHook object
+        if isinstance(attr_value, FunctionHook):
+            attr_value = attr_value.function or attr_value.async_function
+
+        if not callable(attr_value):
+            raise DeserializationError(f"The final attribute is not callable: {attr_value}")
+
+        # `builtins` is on the allowlist (for `builtins.print` etc.), so the module check
+        # above does not stop dangerous builtins like `eval`/`exec` from resolving here. Block them.
+        _check_not_denied_builtin(attr_value, callable_handle)
+
+        return attr_value
+
+    # Fallback if we never find anything
+    raise DeserializationError(f"Could not import '{callable_handle}' as a module or callable.")

haystack/utils/dataclasses.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import warnings
+from functools import wraps
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+def _warn_on_inplace_mutation(cls: T) -> T:
+    """
+    Decorator that warns if the dataclass is mutated in-place.
+    """
+    initializing = set()
+
+    # mypy requires using getattr/setattr for dunder access, but ruff prefers
+    # direct attribute access. We silence mypy here in favor of the more explicit syntax.
+    original_init = cls.__init__  # type: ignore[misc]
+    original_setattr = cls.__setattr__
+
+    @wraps(original_init)
+    def __init_track__(self: T, *args: Any, **kwargs: Any) -> None:
+        # We don't raise warnings during initialization, i.e. during the first call to __init__ and __post_init__.
+        initializing.add(id(self))
+        try:
+            return original_init(self, *args, **kwargs)
+        finally:
+            initializing.discard(id(self))
+
+    @wraps(original_setattr)
+    def __setattr_warn__(self: T, name: str, value: Any) -> None:
+        # We raise warnings if the dataclass is mutated in-place after initialization.
+        if (
+            id(self) not in initializing
+            and name in getattr(self, "__dataclass_fields__", {})
+            and name in getattr(self, "__dict__", {})
+        ):
+            # We raise a warning if the attribute is a dataclass field and a dictionary key.
+            warnings.warn(
+                f"Mutating attribute '{name}' on an instance of "
+                f"'{type(self).__name__}' can lead to unexpected behavior by affecting other parts of the pipeline "
+                "that use the same dataclass instance. "
+                f"Use `dataclasses.replace(instance, {name}=new_value)` instead. "
+                "See https://docs.haystack.deepset.ai/docs/custom-components#requirements for details.",
+                Warning,
+                stacklevel=2,
+            )
+        # mypy infers original_setattr as bound to the type, expecting (str, Any), we call the unbound form
+        return original_setattr(self, name, value)  # type: ignore[call-arg, arg-type]
+
+    # mypy considers direct dunder access on a class unsound, ruff prefers direct access
+    cls.__init__ = __init_track__  # type: ignore[misc]
+    # mypy does not allow assigning to a method, ruff prefers direct access
+    cls.__setattr__ = __setattr_warn__  # type: ignore[method-assign, assignment]
+    return cls

haystack/utils/deserialization.py

@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack.core.errors import DeserializationError
+from haystack.core.serialization import component_from_dict, import_class_by_name
+
+
+def deserialize_chatgenerator_inplace(data: dict[str, Any], key: str = "chat_generator") -> None:
+    """
+    Deserialize a ChatGenerator in a dictionary inplace.
+
+    :param data:
+        The dictionary with the serialized data.
+    :param key:
+        The key in the dictionary where the ChatGenerator is stored.
+
+    :raises DeserializationError:
+        If the key is missing in the serialized data, the value is not a dictionary,
+        the type key is missing, the class cannot be imported, or the class lacks a 'from_dict' method.
+    """
+    deserialize_component_inplace(data, key=key)
+
+
+def deserialize_component_inplace(data: dict[str, Any], key: str = "chat_generator") -> None:
+    """
+    Deserialize a Component in a dictionary inplace.
+
+    :param data:
+        The dictionary with the serialized data.
+    :param key:
+        The key in the dictionary where the Component is stored. Default is "chat_generator".
+
+    :raises DeserializationError:
+        If the key is missing in the serialized data, the value is not a dictionary,
+        the type key is missing, the class cannot be imported, or the class lacks a 'from_dict' method.
+    """
+    if key not in data:
+        raise DeserializationError(f"Missing '{key}' in serialization data")
+
+    serialized_component = data[key]
+
+    if not isinstance(serialized_component, dict):
+        raise DeserializationError(f"The value of '{key}' is not a dictionary")
+
+    if "type" not in serialized_component:
+        raise DeserializationError(f"Missing 'type' in {key} serialization data")
+
+    try:
+        component_class = import_class_by_name(serialized_component["type"])
+    except ImportError as e:
+        raise DeserializationError(f"Class '{serialized_component['type']}' not correctly imported") from e
+
+    data[key] = component_from_dict(cls=component_class, data=serialized_component, name=key)

haystack/utils/device.py

@@ -0,0 +1,544 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Optional, Union
+
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(
+    message="PyTorch must be installed to use torch.device or use GPU support in HuggingFace transformers. "
+    "Run 'pip install \"transformers[torch]\"'"
+) as torch_import:
+    import torch
+
+
+class DeviceType(Enum):
+    """
+    Represents device types supported by Haystack.
+
+    This also includes devices that are not directly used by models - for example, the disk device is exclusively used
+    in device maps for frameworks that support offloading model weights to disk.
+    """
+
+    CPU = "cpu"
+    GPU = "cuda"
+    DISK = "disk"
+    MPS = "mps"
+    XPU = "xpu"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "DeviceType":
+        """
+        Create a device type from a string.
+
+        :param string:
+            The string to convert.
+        :returns:
+            The device type.
+        """
+        mapping = {e.value: e for e in DeviceType}
+        _type = mapping.get(string)
+        if _type is None:
+            raise ValueError(f"Unknown device type string '{string}'")
+        return _type
+
+
+@dataclass
+class Device:
+    """
+    A generic representation of a device.
+
+    :param type:
+        The device type.
+    :param id:
+        The optional device id.
+    """
+
+    type: DeviceType
+    id: int | None = field(default=None)
+
+    def __init__(self, type: DeviceType, id: int | None = None) -> None:  # noqa:A002
+        """
+        Create a generic device.
+
+        :param type:
+            The device type.
+        :param id:
+            The device id.
+        """
+        if id is not None and id < 0:
+            raise ValueError(f"Device id must be >= 0, got {id}")
+
+        self.type = type
+        self.id = id
+
+    def __str__(self) -> str:
+        if self.id is None:
+            return str(self.type)
+        return f"{self.type}:{self.id}"
+
+    @staticmethod
+    def cpu() -> "Device":
+        """
+        Create a generic CPU device.
+
+        :returns:
+            The CPU device.
+        """
+        return Device(DeviceType.CPU)
+
+    @staticmethod
+    def gpu(id: int = 0) -> "Device":  # noqa:A002
+        """
+        Create a generic GPU device.
+
+        :param id:
+            The GPU id.
+        :returns:
+            The GPU device.
+        """
+        return Device(DeviceType.GPU, id)
+
+    @staticmethod
+    def disk() -> "Device":
+        """
+        Create a generic disk device.
+
+        :returns:
+            The disk device.
+        """
+        return Device(DeviceType.DISK)
+
+    @staticmethod
+    def mps() -> "Device":
+        """
+        Create a generic Apple Metal Performance Shader device.
+
+        :returns:
+            The MPS device.
+        """
+        return Device(DeviceType.MPS)
+
+    @staticmethod
+    def xpu() -> "Device":
+        """
+        Create a generic Intel GPU Optimization device.
+
+        :returns:
+            The XPU device.
+        """
+        return Device(DeviceType.XPU)
+
+    @staticmethod
+    def from_str(string: str) -> "Device":
+        """
+        Create a generic device from a string.
+
+        :returns:
+            The device.
+
+        """
+        device_type_str, device_id = _split_device_string(string)
+        return Device(DeviceType.from_str(device_type_str), device_id)
+
+
+@dataclass
+class DeviceMap:
+    """
+    A generic mapping from strings to devices.
+
+    The semantics of the strings are dependent on target framework. Primarily used to deploy HuggingFace models to
+    multiple devices.
+
+    :param mapping:
+        Dictionary mapping strings to devices.
+    """
+
+    mapping: dict[str, Device] = field(default_factory=dict, hash=False)
+
+    def __getitem__(self, key: str) -> Device:
+        return self.mapping[key]
+
+    def __setitem__(self, key: str, value: Device) -> None:
+        self.mapping[key] = value
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.mapping
+
+    def __len__(self) -> int:
+        return len(self.mapping)
+
+    def __iter__(self) -> Iterator[tuple[str, Device]]:
+        return iter(self.mapping.items())
+
+    def to_dict(self) -> dict[str, str]:
+        """
+        Serialize the mapping to a JSON-serializable dictionary.
+
+        :returns:
+            The serialized mapping.
+        """
+        return {key: str(device) for key, device in self.mapping.items()}
+
+    @property
+    def first_device(self) -> Device | None:
+        """
+        Return the first device in the mapping, if any.
+
+        :returns:
+            The first device.
+        """
+        if not self.mapping:
+            return None
+        return next(iter(self.mapping.values()))
+
+    @staticmethod
+    def from_dict(dict: dict[str, str]) -> "DeviceMap":  # noqa:A002
+        """
+        Create a generic device map from a JSON-serialized dictionary.
+
+        :param dict:
+            The serialized mapping.
+        :returns:
+            The generic device map.
+        """
+        mapping = {}
+        for key, device_str in dict.items():
+            mapping[key] = Device.from_str(device_str)
+        return DeviceMap(mapping)
+
+    @staticmethod
+    def from_hf(hf_device_map: dict[str, Union[int, str, "torch.device"]]) -> "DeviceMap":
+        """
+        Create a generic device map from a HuggingFace device map.
+
+        :param hf_device_map:
+            The HuggingFace device map.
+        :returns:
+            The deserialized device map.
+        :raises TypeError: If a device value in the map is not an int, str, or torch.device.
+        """
+        mapping = {}
+        for key, device in hf_device_map.items():
+            if isinstance(device, int):
+                mapping[key] = Device(DeviceType.GPU, device)
+            elif isinstance(device, str):
+                device_type, device_id = _split_device_string(device)
+                mapping[key] = Device(DeviceType.from_str(device_type), device_id)
+            elif isinstance(device, torch.device):
+                device_type = device.type
+                device_id = device.index
+                mapping[key] = Device(DeviceType.from_str(device_type), device_id)
+            else:
+                raise TypeError(
+                    f"Couldn't convert HuggingFace device map - unexpected device '{str(device)}' for '{key}'"
+                )
+        return DeviceMap(mapping)
+
+
+@dataclass(frozen=True)
+class ComponentDevice:
+    """
+    A representation of a device for a component.
+
+    This can be either a single device or a device map.
+    """
+
+    _single_device: Device | None = field(default=None)
+    _multiple_devices: DeviceMap | None = field(default=None)
+
+    @classmethod
+    def from_str(cls, device_str: str) -> "ComponentDevice":
+        """
+        Create a component device representation from a device string.
+
+        The device string can only represent a single device.
+
+        :param device_str:
+            The device string.
+        :returns:
+            The component device representation.
+        """
+        device = Device.from_str(device_str)
+        return cls.from_single(device)
+
+    @classmethod
+    def from_single(cls, device: Device) -> "ComponentDevice":
+        """
+        Create a component device representation from a single device.
+
+        Disks cannot be used as single devices.
+
+        :param device:
+            The device.
+        :returns:
+            The component device representation.
+        """
+        if device.type == DeviceType.DISK:
+            raise ValueError("The disk device can only be used as a part of device maps")
+
+        return cls(_single_device=device)
+
+    @classmethod
+    def from_multiple(cls, device_map: DeviceMap) -> "ComponentDevice":
+        """
+        Create a component device representation from a device map.
+
+        :param device_map:
+            The device map.
+        :returns:
+            The component device representation.
+        """
+        return cls(_multiple_devices=device_map)
+
+    def _validate(self) -> None:
+        """
+        Validate the component device representation.
+        """
+        if not (self._single_device is not None) ^ (self._multiple_devices is not None):
+            raise ValueError(
+                "The component device can neither be empty nor contain both a single device and a device map"
+            )
+
+    def to_torch(self) -> "torch.device":
+        """
+        Convert the component device representation to PyTorch format.
+
+        Device maps are not supported.
+
+        :returns:
+            The PyTorch device representation.
+        """
+        self._validate()
+
+        if self._single_device is None:
+            raise ValueError("Only single devices can be converted to PyTorch format")
+
+        torch_import.check()
+        assert self._single_device is not None
+        return torch.device(str(self._single_device))
+
+    def to_torch_str(self) -> str:
+        """
+        Convert the component device representation to PyTorch string format.
+
+        Device maps are not supported.
+
+        :returns:
+            The PyTorch device string representation.
+        """
+        self._validate()
+
+        if self._single_device is None:
+            raise ValueError("Only single devices can be converted to PyTorch format")
+
+        assert self._single_device is not None
+        return str(self._single_device)
+
+    def to_spacy(self) -> int:
+        """
+        Convert the component device representation to spaCy format.
+
+        Device maps are not supported.
+
+        :returns:
+            The spaCy device representation.
+        """
+        self._validate()
+
+        if self._single_device is None:
+            raise ValueError("Only single devices can be converted to spaCy format")
+
+        assert self._single_device is not None
+        if self._single_device.type == DeviceType.GPU:
+            assert self._single_device.id is not None
+            return self._single_device.id
+        return -1
+
+    def to_hf(self) -> int | str | dict[str, int | str]:
+        """
+        Convert the component device representation to HuggingFace format.
+
+        :returns:
+            The HuggingFace device representation.
+        """
+        self._validate()
+
+        def convert_device(device: Device, *, gpu_id_only: bool = False) -> int | str:
+            if gpu_id_only and device.type == DeviceType.GPU:
+                assert device.id is not None
+                return device.id
+            return str(device)
+
+        if self._single_device is not None:
+            return convert_device(self._single_device)
+
+        assert self._multiple_devices is not None
+        return {key: convert_device(device, gpu_id_only=True) for key, device in self._multiple_devices.mapping.items()}
+
+    def update_hf_kwargs(self, hf_kwargs: dict[str, Any], *, overwrite: bool) -> dict[str, Any]:
+        """
+        Convert the component device representation to HuggingFace format.
+
+        Add them as canonical keyword arguments to the keyword arguments dictionary.
+
+        :param hf_kwargs:
+            The HuggingFace keyword arguments dictionary.
+        :param overwrite:
+            Whether to overwrite existing device arguments.
+        :returns:
+            The HuggingFace keyword arguments dictionary.
+        """
+        self._validate()
+
+        if not overwrite and any(x in hf_kwargs for x in ("device", "device_map")):
+            return hf_kwargs
+
+        converted = self.to_hf()
+        key = "device_map" if self.has_multiple_devices else "device"
+        hf_kwargs[key] = converted
+        return hf_kwargs
+
+    @property
+    def has_multiple_devices(self) -> bool:
+        """
+        Whether this component device representation contains multiple devices.
+        """
+        self._validate()
+
+        return self._multiple_devices is not None
+
+    @property
+    def first_device(self) -> Optional["ComponentDevice"]:
+        """
+        Return either the single device or the first device in the device map, if any.
+
+        :returns:
+            The first device.
+        """
+        self._validate()
+
+        if self._single_device is not None:
+            return self.from_single(self._single_device)
+
+        assert self._multiple_devices is not None
+        assert self._multiple_devices.first_device is not None
+        return self.from_single(self._multiple_devices.first_device)
+
+    @staticmethod
+    def resolve_device(device: Optional["ComponentDevice"] = None) -> "ComponentDevice":
+        """
+        Select a device for a component. If a device is specified, it's used. Otherwise, the default device is used.
+
+        :param device:
+            The provided device, if any.
+        :returns:
+            The resolved device.
+        """
+        if not isinstance(device, ComponentDevice) and device is not None:
+            raise ValueError(
+                f"Invalid component device type '{type(device).__name__}'. Must either be None or ComponentDevice."
+            )
+
+        if device is None:
+            device = ComponentDevice.from_single(_get_default_device())
+
+        return device
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Convert the component device representation to a JSON-serializable dictionary.
+
+        :returns:
+            The dictionary representation.
+        """
+        if self._single_device is not None:
+            return {"type": "single", "device": str(self._single_device)}
+        if self._multiple_devices is not None:
+            return {"type": "multiple", "device_map": self._multiple_devices.to_dict()}
+        # Unreachable
+        raise AssertionError()
+
+    @classmethod
+    def from_dict(cls, dict: dict[str, Any]) -> "ComponentDevice":  # noqa:A002
+        """
+        Create a component device representation from a JSON-serialized dictionary.
+
+        :param dict:
+            The serialized representation.
+        :returns:
+            The deserialized component device.
+        """
+        if dict["type"] == "single":
+            return cls.from_str(dict["device"])
+        if dict["type"] == "multiple":
+            return cls.from_multiple(DeviceMap.from_dict(dict["device_map"]))
+        raise ValueError(f"Unknown component device type '{dict['type']}' in serialized data")
+
+
+def _get_default_device() -> Device:
+    """
+    Return the default device for Haystack.
+
+    Precedence:
+        GPU > XPU > MPS > CPU. If PyTorch is not installed, only CPU is available.
+
+    :returns:
+        The default device.
+    """
+    try:
+        torch_import.check()
+
+        has_mps = (
+            hasattr(torch.backends, "mps")
+            and torch.backends.mps.is_available()
+            and os.getenv("HAYSTACK_MPS_ENABLED", "true") != "false"
+        )
+        has_cuda = torch.cuda.is_available()
+        has_xpu = (
+            hasattr(torch, "xpu")
+            and hasattr(torch.xpu, "is_available")
+            and torch.xpu.is_available()
+            and os.getenv("HAYSTACK_XPU_ENABLED", "true") != "false"
+        )
+    except ImportError:
+        has_mps = False
+        has_cuda = False
+        has_xpu = False
+
+    if has_cuda:
+        return Device.gpu()
+    if has_xpu:
+        return Device.xpu()
+    if has_mps:
+        return Device.mps()
+    return Device.cpu()
+
+
+def _split_device_string(string: str) -> tuple[str, int | None]:
+    """
+    Split a device string into device type and device id.
+
+    :param string:
+        The device string to split.
+    :returns:
+        The device type and device id, if any.
+    """
+    if ":" in string:
+        device_type, device_id_str = string.split(":")
+        try:
+            device_id = int(device_id_str)
+        except ValueError as e:
+            raise ValueError(f"Device id must be an integer, got {device_id_str}") from e
+    else:
+        device_type = string
+        device_id = None
+    return device_type, device_id

haystack/utils/experimental.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import functools
+import warnings
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+def _experimental(cls: type[T]) -> type[T]:
+    """
+    Class decorator that marks a Haystack component as experimental.
+
+    Components decorated with @experimental are subject to breaking changes
+    or removal in future releases without prior deprecation notice.
+
+    ## Usage example
+
+        @_experimental
+        @component
+        class MyComponent:
+            ...
+    """
+    # getattr/setattr are intentional here: direct attribute access (cls.__init__, cls.__init__ = ...)
+    # triggers mypy [misc] and [attr-defined] errors because T is an unbound TypeVar.
+    # noqa comments suppress ruff B009/B010 which would auto-revert these back to direct access.
+    original_init: Any = getattr(cls, "__init__")  # noqa: B009
+
+    @functools.wraps(original_init)
+    def new_init(self: Any, *args: Any, **kwargs: Any) -> None:
+        warnings.warn(
+            f"'{cls.__name__}' is an experimental component and may change or be removed "
+            "in future releases without prior deprecation notice. ",
+            ExperimentalWarning,
+            stacklevel=2,
+        )
+        original_init(self, *args, **kwargs)
+
+    setattr(cls, "__init__", new_init)  # noqa: B010
+    setattr(cls, "__experimental__", True)  # noqa: B010
+    return cls
+
+
+class ExperimentalWarning(UserWarning):
+    """Warning emitted when an experimental Haystack component is instantiated."""

haystack/utils/filters.py

@@ -0,0 +1,217 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from dataclasses import fields
+from datetime import datetime
+from typing import Any
+
+import dateutil.parser
+
+from haystack.dataclasses import ByteStream, Document
+from haystack.errors import FilterError
+
+
+def document_matches_filter(filters: dict[str, Any], document: Document | ByteStream) -> bool:
+    """
+    Return whether `filters` match the Document or the ByteStream.
+
+    For a detailed specification of the filters, refer to the
+    `DocumentStore.filter_documents()` protocol documentation.
+    """
+    if "field" in filters:
+        return _comparison_condition(condition=filters, document=document)
+    return _logic_condition(condition=filters, document=document)
+
+
+def _and(document: Document | ByteStream, conditions: list[dict[str, Any]]) -> bool:
+    return all(_comparison_condition(condition=condition, document=document) for condition in conditions)
+
+
+def _or(document: Document | ByteStream, conditions: list[dict[str, Any]]) -> bool:
+    return any(_comparison_condition(condition=condition, document=document) for condition in conditions)
+
+
+def _not(document: Document | ByteStream, conditions: list[dict[str, Any]]) -> bool:
+    return not _and(document=document, conditions=conditions)
+
+
+LOGICAL_OPERATORS = {"NOT": _not, "OR": _or, "AND": _and}
+
+
+def _equal(value: Any, filter_value: Any) -> bool:
+    return value == filter_value
+
+
+def _not_equal(value: Any, filter_value: Any) -> bool:
+    return not _equal(value=value, filter_value=filter_value)
+
+
+def _prepare_ordering_comparison(value: Any, filter_value: Any) -> tuple[Any, Any]:
+    """Normalize both values for ordering comparisons, parsing strings as dates."""
+    if isinstance(value, str) or isinstance(filter_value, str):
+        if not isinstance(value, datetime):
+            value = _parse_date(value)
+        if not isinstance(filter_value, datetime):
+            filter_value = _parse_date(filter_value)
+
+    if isinstance(value, datetime) and isinstance(filter_value, datetime):
+        value, filter_value = _ensure_both_dates_naive_or_aware(value, filter_value)
+
+    if isinstance(filter_value, list):
+        msg = f"Filter value can't be of type {type(filter_value)} using operators '>', '>=', '<', '<='"
+        raise FilterError(msg)
+    return value, filter_value
+
+
+def _greater_than(value: Any, filter_value: Any) -> bool:
+    if value is None or filter_value is None:
+        # We can't compare None values reliably using operators '>', '>=', '<', '<='
+        return False
+
+    value, filter_value = _prepare_ordering_comparison(value=value, filter_value=filter_value)
+    return value > filter_value
+
+
+def _parse_date(value: str) -> datetime:
+    """Try parsing the value as an ISO format date, then fall back to dateutil.parser."""
+    try:
+        return datetime.fromisoformat(value)
+    except (ValueError, TypeError):
+        try:
+            return dateutil.parser.parse(value)
+        except (ValueError, TypeError) as exc:
+            msg = (
+                "Can't compare strings using operators '>', '>=', '<', '<='. "
+                "Strings are only comparable if they are ISO formatted dates."
+            )
+            raise FilterError(msg) from exc
+
+
+def _ensure_both_dates_naive_or_aware(date1: datetime, date2: datetime) -> tuple[datetime, datetime]:
+    """Ensure that both dates are either naive or aware."""
+    # Both naive
+    if date1.tzinfo is None and date2.tzinfo is None:
+        return date1, date2
+
+    # Both aware
+    if date1.tzinfo is not None and date2.tzinfo is not None:
+        return date1, date2
+
+    # One naive, one aware
+    if date1.tzinfo is None:
+        date1 = date1.replace(tzinfo=date2.tzinfo)
+    else:
+        date2 = date2.replace(tzinfo=date1.tzinfo)
+    return date1, date2
+
+
+def _greater_than_equal(value: Any, filter_value: Any) -> bool:
+    if value is None or filter_value is None:
+        # We can't compare None values reliably using operators '>', '>=', '<', '<='
+        return False
+
+    value, filter_value = _prepare_ordering_comparison(value=value, filter_value=filter_value)
+    return value >= filter_value
+
+
+def _less_than(value: Any, filter_value: Any) -> bool:
+    if value is None or filter_value is None:
+        # We can't compare None values reliably using operators '>', '>=', '<', '<='
+        return False
+
+    value, filter_value = _prepare_ordering_comparison(value=value, filter_value=filter_value)
+    return value < filter_value
+
+
+def _less_than_equal(value: Any, filter_value: Any) -> bool:
+    if value is None or filter_value is None:
+        # We can't compare None values reliably using operators '>', '>=', '<', '<='
+        return False
+
+    value, filter_value = _prepare_ordering_comparison(value=value, filter_value=filter_value)
+    return value <= filter_value
+
+
+def _in(value: Any, filter_value: Any) -> bool:
+    if not isinstance(filter_value, list):
+        msg = (
+            f"Filter value must be a `list` when using operator 'in' or 'not in', received type '{type(filter_value)}'"
+        )
+        raise FilterError(msg)
+    return any(_equal(e, value) for e in filter_value)
+
+
+def _not_in(value: Any, filter_value: Any) -> bool:
+    return not _in(value=value, filter_value=filter_value)
+
+
+COMPARISON_OPERATORS = {
+    "==": _equal,
+    "!=": _not_equal,
+    ">": _greater_than,
+    ">=": _greater_than_equal,
+    "<": _less_than,
+    "<=": _less_than_equal,
+    "in": _in,
+    "not in": _not_in,
+}
+
+
+def _logic_condition(condition: dict[str, Any], document: Document | ByteStream) -> bool:
+    if "operator" not in condition:
+        msg = f"'operator' key missing in {condition}"
+        raise FilterError(msg)
+    if "conditions" not in condition:
+        msg = f"'conditions' key missing in {condition}"
+        raise FilterError(msg)
+    operator: str = condition["operator"]
+    if operator not in LOGICAL_OPERATORS:
+        msg = f"Unknown logical operator '{operator}'. Valid operators are: {sorted(LOGICAL_OPERATORS)}"
+        raise FilterError(msg)
+    conditions: list[dict[str, Any]] = condition["conditions"]
+    return LOGICAL_OPERATORS[operator](document=document, conditions=conditions)
+
+
+def _comparison_condition(condition: dict[str, Any], document: Document | ByteStream) -> bool:
+    if "field" not in condition:
+        # 'field' key is only found in comparison dictionaries.
+        # We assume this is a logic dictionary since it's not present.
+        return _logic_condition(condition=condition, document=document)
+    field: str = condition["field"]
+
+    if "operator" not in condition:
+        msg = f"'operator' key missing in {condition}"
+        raise FilterError(msg)
+    if "value" not in condition:
+        msg = f"'value' key missing in {condition}"
+        raise FilterError(msg)
+
+    if "." in field:
+        # Handles fields formatted like so:
+        # 'meta.person.name'
+        parts = field.split(".")
+        document_value = getattr(document, parts[0])
+        for part in parts[1:]:
+            if not isinstance(document_value, dict) or part not in document_value:
+                # If a field is not found (or an intermediate value is not a dict,
+                # e.g. None) we treat it as None
+                document_value = None
+                break
+            document_value = document_value[part]
+    elif field not in [f.name for f in fields(document)]:
+        # Converted legacy filters don't add the `meta.` prefix, so we assume
+        # that all filter fields that are not actual fields in Document are converted
+        # filters.
+        #
+        # We handle this to avoid breaking compatibility with converted legacy filters.
+        # This will be removed as soon as we stop supporting legacy filters.
+        document_value = document.meta.get(field)
+    else:
+        document_value = getattr(document, field)
+    operator: str = condition["operator"]
+    if operator not in COMPARISON_OPERATORS:
+        msg = f"Unknown comparison operator '{operator}'. Valid operators are: {sorted(COMPARISON_OPERATORS)}"
+        raise FilterError(msg)
+    filter_value: Any = condition["value"]
+    return COMPARISON_OPERATORS[operator](filter_value=filter_value, value=document_value)

haystack/utils/hf.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import logging
+from haystack.dataclasses import ChatMessage, ImageContent, ReasoningContent, TextContent
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(message="Run 'pip install \"transformers[torch]\"'") as torch_import:
+    import torch
+
+logger = logging.getLogger(__name__)
+
+
+def serialize_hf_model_kwargs(kwargs: dict[str, Any]) -> None:
+    """
+    Recursively serialize HuggingFace specific model keyword arguments in-place to make them JSON serializable.
+
+    :param kwargs: The keyword arguments to serialize
+    """
+    torch_import.check()
+
+    for k, v in kwargs.items():
+        # torch.dtype
+        if isinstance(v, torch.dtype):
+            kwargs[k] = str(v)
+
+        if isinstance(v, dict):
+            serialize_hf_model_kwargs(v)
+
+
+def deserialize_hf_model_kwargs(kwargs: dict[str, Any]) -> None:
+    """
+    Recursively deserialize HuggingFace specific model keyword arguments in-place to make them JSON serializable.
+
+    :param kwargs: The keyword arguments to deserialize
+    """
+    torch_import.check()
+
+    for k, v in kwargs.items():
+        # torch.dtype
+        if isinstance(v, str) and v.startswith("torch."):
+            dtype_str = v.split(".")[1]
+            dtype = getattr(torch, dtype_str, None)
+            if dtype is not None and isinstance(dtype, torch.dtype):
+                kwargs[k] = dtype
+
+        if isinstance(v, dict):
+            deserialize_hf_model_kwargs(v)
+
+
+def convert_message_to_hf_format(message: ChatMessage) -> dict[str, Any]:
+    """
+    Convert a message to the format expected by Hugging Face.
+
+    Note: ReasoningContent is skipped during conversion because the HuggingFace Inference API
+    (which follows the OpenAI-compatible chat completion format) does not support reasoning
+    in input messages. Reasoning is captured from model outputs for transparency but is not
+    sent back to the API in multi-turn conversations.
+    """
+    text_contents = message.texts
+    tool_calls = message.tool_calls
+    tool_call_results = message.tool_call_results
+    images = message.images
+
+    # Filter out ReasoningContent from the content list for validation
+    # ReasoningContent is for human transparency only, not sent to the API
+    non_reasoning_content = [c for c in message._content if not isinstance(c, ReasoningContent)]
+
+    if not text_contents and not tool_calls and not tool_call_results and not images:
+        raise ValueError(
+            "A `ChatMessage` must contain at least one `TextContent`, `ToolCall`, `ToolCallResult`, or `ImageContent`."
+        )
+    if len(tool_call_results) > 0 and len(non_reasoning_content) > 1:
+        raise ValueError(
+            "For compatibility with the Hugging Face API, a `ChatMessage` with a `ToolCallResult` "
+            "cannot contain any other content."
+        )
+
+    # HF always expects a content field, even if it is empty
+    hf_msg: dict[str, Any] = {"role": message._role.value, "content": ""}
+
+    if tool_call_results:
+        result = tool_call_results[0]
+        hf_msg["content"] = result.result
+        if tc_id := result.origin.id:
+            hf_msg["tool_call_id"] = tc_id
+        # HF does not provide a way to communicate errors in tool invocations, so we ignore the error field
+        return hf_msg
+
+    # Handle multimodal content (text + images) preserving order
+    if text_contents or images:
+        content_parts: list[dict[str, Any]] = []
+        for part in message._content:
+            if isinstance(part, TextContent):
+                content_parts.append({"type": "text", "text": part.text})
+            elif isinstance(part, ImageContent):
+                image_url = f"data:{part.mime_type or 'image/jpeg'};base64,{part.base64_image}"
+                content_parts.append({"type": "image_url", "image_url": {"url": image_url}})
+
+        if len(content_parts) == 1 and not images:
+            # content is a string
+            hf_msg["content"] = content_parts[0]["text"]
+        else:
+            hf_msg["content"] = content_parts
+
+    if tool_calls:
+        hf_tool_calls = []
+        for tc in tool_calls:
+            hf_tool_call = {"type": "function", "function": {"name": tc.tool_name, "arguments": tc.arguments}}
+            if tc.id is not None:
+                hf_tool_call["id"] = tc.id
+            hf_tool_calls.append(hf_tool_call)
+        hf_msg["tool_calls"] = hf_tool_calls
+
+    return hf_msg

haystack/utils/http_client.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Literal, overload
+
+import httpx
+
+
+@overload
+def init_http_client(
+    http_client_kwargs: dict[str, Any] | None = ..., async_client: Literal[False] = ...
+) -> httpx.Client | None: ...
+@overload
+def init_http_client(
+    http_client_kwargs: dict[str, Any] | None = ..., async_client: Literal[True] = ...
+) -> httpx.AsyncClient | None: ...
+def init_http_client(
+    http_client_kwargs: dict[str, Any] | None = None, async_client: bool = False
+) -> httpx.Client | httpx.AsyncClient | None:
+    """
+    Initialize an httpx client based on the http_client_kwargs.
+
+    :param http_client_kwargs:
+        The kwargs to pass to the httpx client.
+    :param async_client:
+        Whether to initialize an async client.
+
+    :returns:
+        A httpx client or an async httpx client.
+    """
+    if not http_client_kwargs:
+        return None
+    if not isinstance(http_client_kwargs, dict):
+        raise TypeError("The parameter 'http_client_kwargs' must be a dictionary.")
+
+    # Create a copy to avoid modifying the original dict
+    processed_kwargs = http_client_kwargs.copy()
+
+    # Handle limits parameter - convert dict to httpx.Limits object if needed
+    if "limits" in processed_kwargs and isinstance(processed_kwargs["limits"], dict):
+        limits_dict = processed_kwargs["limits"]
+        processed_kwargs["limits"] = httpx.Limits(**limits_dict)
+
+    if async_client:
+        return httpx.AsyncClient(**processed_kwargs)
+    return httpx.Client(**processed_kwargs)

haystack/utils/jinja2_chat_extension.py

@@ -0,0 +1,419 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import secrets
+from collections.abc import Callable
+from typing import Any
+
+from jinja2 import TemplateSyntaxError, nodes, pass_environment
+from jinja2.ext import Extension
+from markupsafe import Markup
+
+from haystack import logging
+from haystack.dataclasses.chat_message import (
+    ChatMessage,
+    ChatMessageContentT,
+    ChatRole,
+    FileContent,
+    ImageContent,
+    ReasoningContent,
+    TextContent,
+    ToolCall,
+    ToolCallResult,
+    _deserialize_content_part,
+    _serialize_content_part,
+)
+
+logger = logging.getLogger(__name__)
+
+_TAG_NAME = "haystack_content_part"
+_NONCE_ATTR = "_haystack_content_part_nonce"
+
+
+def _sentinel_tags(nonce: str) -> tuple[str, str]:
+    """Build the (opening, closing) sentinel tags for the given nonce"""
+    return f"<{_TAG_NAME}:{nonce}>", f"</{_TAG_NAME}:{nonce}>"
+
+
+class _TemplatizedPart(Markup):
+    """Marker type for content produced by `templatize_part`."""
+
+    pass
+
+
+def _finalize(value: object) -> str:
+    """
+    Jinja2 `finalize` callback that prevents sentinel tag injection.
+
+    Called automatically on every `{{ }}` expression result during template rendering.
+    Legitimate structured content from the `templatize_part` filter is wrapped in `_TemplatizedPart` and passes.
+    Any other value containing sentinel tags has those tags replaced with harmless HTML entities so that
+    `_parse_content_parts` will not treat them as structured content.
+    """
+    if isinstance(value, _TemplatizedPart):
+        return value
+
+    # escape the leading "<" of any sentinel tag so that `_parse_content_parts` cannot recognize it
+    return str(value).replace(f"<{_TAG_NAME}", f"&lt;{_TAG_NAME}").replace(f"</{_TAG_NAME}", f"&lt;/{_TAG_NAME}")
+
+
+def _redact_nonce(text: str, start_tag: str, end_tag: str) -> str:
+    return text.replace(start_tag, f"<{_TAG_NAME}:redacted>").replace(end_tag, f"</{_TAG_NAME}:redacted>")
+
+
+class ChatMessageExtension(Extension):
+    """
+    A Jinja2 extension for creating structured chat messages with mixed content types.
+
+    This extension provides a custom `{% message %}` tag that allows creating chat messages
+    with different attributes (role, name, meta) and mixed content types (text, images, etc.).
+
+    Inspired by [Banks](https://github.com/masci/banks).
+
+    Example:
+    ```
+    {% message role="system" %}
+    You are a helpful assistant. You like to talk with {{user_name}}.
+    {% endmessage %}
+
+    {% message role="user" %}
+    Hello! I am {{user_name}}. Please describe the images.
+    {% for image in images %}
+    {{ image | templatize_part }}
+    {% endfor %}
+    {% endmessage %}
+    ```
+
+    This extension also provides an `{% insert %}` placeholder tag that evaluates an expression to a `ChatMessage`
+    or a list of `ChatMessage` objects and expands it into the prompt, so a runtime conversation can be interleaved
+    with literal `{% message %}` blocks:
+
+    ```
+    {% message role="system" %}You are a helpful assistant.{% endmessage %}
+    {% insert messages %}
+    {% message role="user" %}{{ query }}{% endmessage %}
+    ```
+
+    The expression can be a plain variable (`{% insert messages %}`), a slice or index
+    (`{% insert messages[-1:] %}`, `{% insert messages[-1] %}`), or a combination of variables
+    (`{% insert previous + current %}`).
+
+    ### How it works
+    1. The `{% message %}` tag is used to define a chat message.
+    2. The message can contain text and other structured content parts.
+    3. To include a structured content part in the message, the `| templatize_part` filter is used.
+       The filter serializes the content part into a JSON string and wraps it in a `<haystack_content_part>` tag.
+    4. The `_build_chat_message_json` method of the extension parses the message content parts,
+       converts them into a ChatMessage object and serializes it to a JSON string.
+    5. The obtained JSON string is usable in the ChatPromptBuilder component, where templates are rendered to actual
+       ChatMessage objects.
+    """
+
+    SUPPORTED_ROLES = [role.value for role in ChatRole]
+
+    tags = {"message", "insert"}
+
+    def __init__(self, environment: Any) -> None:
+        super().__init__(environment)
+        # a fresh random nonce per environment to produce sentinel tags only usable by `templatize_part`
+        setattr(environment, _NONCE_ATTR, secrets.token_hex(16))
+        # values not produced by `templatize_part` get their sentinel-like tags escaped
+        environment.finalize = _finalize
+        environment.filters["templatize_part"] = templatize_part
+
+    def parse(self, parser: Any) -> nodes.Node | list[nodes.Node]:
+        """
+        Dispatch parsing based on the tag that triggered the extension.
+
+        Handles both the single `{% message %}` block tag and the `{% insert %}` placeholder tag.
+
+        :param parser: The Jinja2 parser instance
+        :return: A CallBlock node containing the parsed configuration
+        """
+        tag = next(parser.stream)
+        if tag.value == "insert":
+            return self._parse_insert_tag(parser, tag.lineno)
+        return self._parse_message_tag(parser, tag.lineno)
+
+    def _parse_insert_tag(self, parser: Any, lineno: int) -> nodes.Node:
+        """
+        Parse the `{% insert %}` placeholder tag.
+
+        This bodyless tag evaluates an expression to a `ChatMessage` or a list of `ChatMessage` objects and expands
+        it into the same JSON-line format produced by `{% message %}` blocks, so messages provided at runtime can be
+        interleaved with literal message blocks (for example a system message above and a user message below).
+
+        The expression can be a plain variable (`{% insert messages %}`), a slice or index
+        (`{% insert messages[-1:] %}`, `{% insert messages[-1] %}`), or a combination of variables
+        (`{% insert previous + current %}`).
+
+        :param parser: The Jinja2 parser instance
+        :param lineno: The line number of the tag, used for error reporting.
+        :return: A CallBlock node that expands the evaluated expression.
+        :raises TemplateSyntaxError: If the tag is not given an expression.
+        """
+        if parser.stream.current.test("block_end"):
+            raise TemplateSyntaxError(
+                "The 'insert' tag requires an expression that evaluates to a ChatMessage or a list of ChatMessage "
+                "objects, for example '{% insert messages %}' or '{% insert messages[-1:] %}'.",
+                lineno,
+            )
+        expr = parser.parse_expression()
+        # Bodyless tag: empty body, no matching end tag required.
+        return nodes.CallBlock(
+            self.call_method(name="_build_inserted_messages_json", args=[expr]), [], [], []
+        ).set_lineno(lineno)
+
+    def _parse_message_tag(self, parser: Any, lineno: int) -> nodes.Node | list[nodes.Node]:
+        """
+        Parse the message tag and its attributes in the Jinja2 template.
+
+        This method handles the parsing of role (mandatory), name (optional), meta (optional) and message body content.
+
+        :param parser: The Jinja2 parser instance
+        :param lineno: The line number of the tag, used for error reporting.
+        :return: A CallBlock node containing the parsed message configuration
+        :raises TemplateSyntaxError: If an invalid role is provided
+        """
+
+        # Parse role attribute (mandatory)
+        parser.stream.expect("name:role")
+        parser.stream.expect("assign")
+        role_expr = parser.parse_expression()
+
+        if isinstance(role_expr, nodes.Const):
+            role = role_expr.value
+            if role not in self.SUPPORTED_ROLES:
+                raise TemplateSyntaxError(f"Role must be one of: {', '.join(self.SUPPORTED_ROLES)}", lineno)
+
+        # Parse optional name attribute
+        name_expr = None
+        if parser.stream.current.test("name:name"):
+            parser.stream.skip()
+            parser.stream.expect("assign")
+            name_expr = parser.parse_expression()
+            if not isinstance(name_expr.value, str):
+                raise TemplateSyntaxError("name must be a string", lineno)
+
+        # Parse optional meta attribute
+        meta_expr = None
+        if parser.stream.current.test("name:meta"):
+            parser.stream.skip()
+            parser.stream.expect("assign")
+            meta_expr = parser.parse_expression()
+            if not isinstance(meta_expr, nodes.Dict):
+                raise TemplateSyntaxError("meta must be a dictionary", lineno)
+
+        # Parse message body
+        body = parser.parse_statements(("name:endmessage",), drop_needle=True)
+
+        # Build message node with all parameters
+        return nodes.CallBlock(
+            self.call_method(
+                name="_build_chat_message_json",
+                args=[role_expr, name_expr or nodes.Const(None), meta_expr or nodes.Dict([])],
+            ),
+            [],
+            [],
+            body,
+        ).set_lineno(lineno)
+
+    def _build_chat_message_json(self, role: str, name: str | None, meta: dict, caller: Callable[[], str]) -> str:
+        """
+        Build a ChatMessage object from template content and serialize it to a JSON string.
+
+        This method is called by Jinja2 when processing a `{% message %}` tag.
+        It takes the rendered content from the template, converts XML blocks into ChatMessageContentT objects,
+        creates a ChatMessage object and serializes it to a JSON string.
+
+        :param role: The role of the message
+        :param name: Optional name for the message sender
+        :param meta: Optional metadata dictionary
+        :param caller: Callable that returns the rendered content
+        :return: A JSON string representation of the ChatMessage object
+        """
+
+        content = caller()
+        start_tag, end_tag = _sentinel_tags(getattr(self.environment, _NONCE_ATTR))
+        parts = self._parse_content_parts(content, start_tag, end_tag)
+        if not parts:
+            raise ValueError(
+                f"Message template produced content that couldn't be parsed into any message parts. "
+                f"Content: {_redact_nonce(content, start_tag, end_tag)!r}"
+            )
+
+        chat_message = self._validate_build_chat_message(parts=parts, role=role, meta=meta, name=name)
+
+        return json.dumps(chat_message.to_dict()) + "\n"
+
+    def _build_inserted_messages_json(
+        self,
+        messages: list[ChatMessage] | ChatMessage,
+        caller: Callable[[], str],  # noqa: ARG002
+    ) -> str:
+        """
+        Expand a list of ChatMessage objects into newline-separated JSON, one message per line.
+
+        This method is called by Jinja2 when processing an `{% insert %}` tag. It produces the same JSON-line format
+        as `_build_chat_message_json`, so the messages are parsed back into ChatMessage objects by the
+        ChatPromptBuilder alongside any literal `{% message %}` blocks. The full `ChatMessage.to_dict()` payload is
+        serialized so that all content types (tool calls, tool call results, images, reasoning, name and meta) round
+        trip without loss.
+
+        :param messages: The value the `{% insert %}` expression evaluated to. A missing or empty value expands to
+            nothing. A single ChatMessage is also accepted, since indexing with an integer (for example
+            `{% insert messages[-1] %}`) yields one message rather than a list. The value is validated at render time
+            because it comes from untrusted template input.
+        :param caller: Callable that returns the (empty) rendered body. Unused.
+        :return: Newline-terminated JSON lines, one per message, or an empty string if there are no messages.
+        :raises ValueError: If the value is not a ChatMessage or a list of ChatMessage objects.
+        """
+        if isinstance(messages, ChatMessage):
+            messages = [messages]
+        if not messages:
+            return ""
+        if not isinstance(messages, (list, tuple)) or not all(isinstance(m, ChatMessage) for m in messages):
+            raise ValueError(
+                "The '{% insert %}' expression must evaluate to a ChatMessage or a list of ChatMessage objects. "
+                f"Got: {type(messages).__name__}."
+            )
+        return "".join(json.dumps(message.to_dict()) + "\n" for message in messages)
+
+    @staticmethod
+    def _parse_content_parts(content: str, start_tag: str, end_tag: str) -> list[ChatMessageContentT]:
+        """
+        Parse a string into a sequence of ChatMessageContentT objects.
+
+        This method handles:
+        - Plain text content, converted to TextContent objects
+        - Structured content parts wrapped in sentinel tags, converted to ChatMessageContentT objects
+
+        :param content: Input string containing mixed text and content parts
+        :param start_tag: The opening sentinel tag (including the nonce)
+        :param end_tag: The closing sentinel tag (including the nonce)
+        :return: A list of ChatMessageContentT objects
+        :raises ValueError: If the content is empty or contains only whitespace characters or if a
+                            `<haystack_content_part>` tag is found without a matching closing tag.
+        """
+        if not content.strip():
+            raise ValueError(
+                f"Message content in template is empty or contains only whitespace characters. "
+                f"Content: {_redact_nonce(content, start_tag, end_tag)!r}"
+            )
+
+        parts: list[ChatMessageContentT] = []
+        cursor = 0
+        total_length = len(content)
+
+        while cursor < total_length:
+            tag_start = content.find(start_tag, cursor)
+
+            if tag_start == -1:
+                # No more tags, add remaining text if any
+                remaining_text = content[cursor:].strip()
+                if remaining_text:
+                    parts.append(TextContent(text=remaining_text))
+                break
+
+            # Add text before tag if any
+            if tag_start > cursor:
+                plain_text = content[cursor:tag_start].strip()
+                if plain_text:
+                    parts.append(TextContent(text=plain_text))
+
+            content_start = tag_start + len(start_tag)
+            tag_end = content.find(end_tag, content_start)
+
+            if tag_end == -1:
+                snippet = _redact_nonce(content, start_tag, end_tag)[tag_start : tag_start + 50]
+                raise ValueError(
+                    f"Found unclosed <haystack_content_part> tag at position {tag_start}. Content: '{snippet}...'"
+                )
+
+            json_content = content[content_start:tag_end]
+            data = json.loads(json_content)
+            parts.append(_deserialize_content_part(data))
+
+            cursor = tag_end + len(end_tag)
+
+        return parts
+
+    @staticmethod
+    def _validate_build_chat_message(
+        parts: list[ChatMessageContentT], role: str, meta: dict, name: str | None = None
+    ) -> ChatMessage:
+        """
+        Validate the parts of a chat message and build a ChatMessage object.
+
+        :param parts: Content parts of the message
+        :param role: The role of the message
+        :param meta: The metadata of the message
+        :param name: The optional name of the message
+        :return: A ChatMessage object
+
+        :raises ValueError: If content parts don't allow to build a valid ChatMessage object or the role is not
+                            supported
+        """
+
+        if role == "user":
+            valid_parts = [part for part in parts if isinstance(part, (TextContent, str, ImageContent, FileContent))]
+            if len(parts) != len(valid_parts):
+                raise ValueError(
+                    "User message must contain only TextContent, string, ImageContent or FileContent parts."
+                )
+            return ChatMessage.from_user(meta=meta, name=name, content_parts=valid_parts)
+
+        if role == "system":
+            if not isinstance(parts[0], TextContent):
+                raise ValueError("System message must contain a text part.")
+            text = parts[0].text
+            if len(parts) > 1:
+                raise ValueError("System message must contain only one text part.")
+            return ChatMessage.from_system(meta=meta, name=name, text=text)
+
+        if role == "assistant":
+            texts = [part.text for part in parts if isinstance(part, TextContent)]
+            tool_calls = [part for part in parts if isinstance(part, ToolCall)]
+            reasoning = [part for part in parts if isinstance(part, ReasoningContent)]
+            if len(texts) > 1:
+                raise ValueError("Assistant message must contain one text part at most.")
+            if len(texts) == 0 and len(tool_calls) == 0:
+                raise ValueError("Assistant message must contain at least one text or tool call part.")
+            if len(parts) > len(texts) + len(tool_calls) + len(reasoning):
+                raise ValueError("Assistant message must contain only text, tool call or reasoning parts.")
+            return ChatMessage.from_assistant(
+                meta=meta,
+                name=name,
+                text=texts[0] if texts else None,
+                tool_calls=tool_calls or None,
+                reasoning=reasoning[0] if reasoning else None,
+            )
+
+        if role == "tool":
+            tool_call_results = [part for part in parts if isinstance(part, ToolCallResult)]
+            if len(tool_call_results) == 0 or len(tool_call_results) > 1 or len(parts) > len(tool_call_results):
+                raise ValueError("Tool message must contain only one tool call result.")
+
+            tool_result = tool_call_results[0].result
+            origin = tool_call_results[0].origin
+            error = tool_call_results[0].error
+
+            return ChatMessage.from_tool(meta=meta, tool_result=tool_result, origin=origin, error=error)
+
+        raise ValueError(f"Unsupported role: {role}")
+
+
+@pass_environment
+def templatize_part(environment: Any, value: ChatMessageContentT) -> "_TemplatizedPart":
+    """
+    Jinja filter to convert a ChatMessageContentT object into a JSON string wrapped in sentinel content tags.
+
+    :param environment: The Jinja2 environment
+    :param value: The ChatMessageContentT object to convert
+    :return: A `_TemplatizedPart` holding a JSON string wrapped in special XML content tags
+    :raises ValueError: If the value is not an instance of ChatMessageContentT
+    """
+    start_tag, end_tag = _sentinel_tags(getattr(environment, _NONCE_ATTR))
+    return _TemplatizedPart(f"{start_tag}{json.dumps(_serialize_content_part(value))}{end_tag}")

haystack/utils/jinja2_extensions.py

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from jinja2 import Environment, meta, nodes
+from jinja2.ext import Extension
+
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(message='Run "pip install arrow>=1.3.0"') as arrow_import:
+    import arrow
+
+
+class Jinja2TimeExtension(Extension):
+    """A Jinja2 extension for formatting dates and times."""
+
+    # Syntax for current date
+    tags = {"now"}
+
+    def __init__(self, environment: Environment) -> None:
+        """
+        Initializes the JinjaTimeExtension object.
+
+        :param environment: The Jinja2 environment to initialize the extension with.
+            It provides the context where the extension will operate.
+        """
+        arrow_import.check()
+        super().__init__(environment)
+
+    @staticmethod
+    def _get_datetime(
+        timezone: str, operator: str | None = None, offset: str | None = None, datetime_format: str | None = None
+    ) -> str:
+        """
+        Get the current datetime based on timezone, apply any offset if provided, and format the result.
+
+        :param timezone: The timezone string (e.g., 'UTC' or 'America/New_York') for which the current
+            time should be fetched.
+        :param operator: The operator ('+' or '-') to apply to the offset (used for adding/subtracting intervals).
+            Defaults to None if no offset is applied, otherwise default is '+'.
+        :param offset: The offset string in the format 'interval=value' (e.g., 'hours=2,days=1') specifying how much
+            to adjust the datetime. The intervals can be any valid interval accepted
+            by Arrow (e.g., hours, days, weeks, months). Defaults to None if no adjustment is needed.
+        :param datetime_format: The format string to use for formatting the output datetime.
+            Defaults to '%Y-%m-%d %H:%M:%S' if not provided.
+        """
+        try:
+            dt = arrow.now(timezone)
+        except Exception as e:
+            raise ValueError(f"Invalid timezone {timezone}: {e}") from e
+
+        if offset and operator:
+            try:
+                # Parse the offset and apply it to the datetime object
+                replace_params: dict[str, Any] = {
+                    interval.strip(): float(operator + value.strip())
+                    for param in offset.split(",")
+                    for interval, value in [param.split("=")]
+                }
+                # Shift the datetime fields based on the parsed offset
+                dt = dt.shift(**replace_params)
+            except (ValueError, AttributeError) as e:
+                raise ValueError(f"Invalid offset or operator {offset}, {operator}: {e}") from e
+
+        # Use the provided format or fallback to the default one
+        datetime_format = datetime_format or "%Y-%m-%d %H:%M:%S"
+
+        return dt.strftime(datetime_format)
+
+    def parse(self, parser: Any) -> nodes.Node | list[nodes.Node]:
+        """
+        Parse the template expression to determine how to handle the datetime formatting.
+
+        :param parser: The parser object that processes the template expressions and manages the syntax tree.
+            It's used to interpret the template's structure.
+        """
+        lineno = next(parser.stream).lineno
+        node = parser.parse_expression()
+        # Check if a custom datetime format is provided after a comma
+        datetime_format = parser.parse_expression() if parser.stream.skip_if("comma") else nodes.Const(None)
+
+        # Default Add when no operator is provided
+        operator = "+" if isinstance(node, nodes.Add) else "-"
+        # Call the _get_datetime method with the appropriate operator and offset, if exist
+        call_method = self.call_method(
+            "_get_datetime",
+            [node.left, nodes.Const(operator), node.right, datetime_format]
+            if isinstance(node, (nodes.Add, nodes.Sub))
+            else [node, nodes.Const(None), nodes.Const(None), datetime_format],
+            lineno=lineno,
+        )
+
+        return nodes.Output([call_method], lineno=lineno)
+
+
+def _collect_assigned_variables(ast: nodes.Template) -> set[str]:
+    """
+    Extract variables assigned within the Jinja2 template AST.
+
+    :param ast: The Jinja2 Abstract Syntax Tree (AST) of the template.
+
+    :returns:
+        A set of variable names that are assigned within the template.
+    """
+    # Collect all variables assigned inside the template via {% set %}
+    assigned_variables = set()
+
+    for node in ast.find_all(nodes.Assign):
+        if isinstance(node.target, nodes.Name):
+            assigned_variables.add(node.target.name)
+        elif isinstance(node.target, (nodes.List, nodes.Tuple)):
+            for name_node in node.target.items:
+                if isinstance(name_node, nodes.Name):
+                    assigned_variables.add(name_node.name)
+
+    return assigned_variables
+
+
+def _extract_template_variables_and_assignments(env: Environment, template: str) -> tuple[set[str], set[str]]:
+    """
+    Extract variables from a Jinja2 template and variables assigned within it.
+
+    :param env: A Jinja2 environment.
+    :param template: A Jinja2 template string.
+    :returns: A tuple of (assigned_variables, template_variables) where:
+        - assigned_variables: Variables assigned within the template (e.g., via {% set %})
+        - template_variables: All undeclared variables used in the template
+    """
+    jinja2_ast = env.parse(template)
+    template_variables = meta.find_undeclared_variables(jinja2_ast)
+    assigned_variables = _collect_assigned_variables(jinja2_ast)
+    return assigned_variables, template_variables

haystack/utils/jupyter.py

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+def is_in_jupyter() -> bool:
+    """
+    Returns `True` if in Jupyter or Google Colab, `False` otherwise.
+    """
+    # Inspired by:
+    # https://github.com/explosion/spaCy/blob/e1249d3722765aaca56f538e830add7014d20e2a/spacy/util.py#L1079
+    try:
+        # We don't need to import `get_ipython` as it's always present in Jupyter notebooks
+        if get_ipython().__class__.__name__ == "ZMQInteractiveShell":  # type: ignore[name-defined]
+            return True  # Jupyter notebook or qtconsole
+        if get_ipython().__class__.__module__ == "google.colab._shell":  # type: ignore[name-defined]
+            return True  # Colab notebook
+    except NameError:
+        pass  # Probably standard Python interpreter
+    return False

haystack/utils/misc.py

@@ -0,0 +1,258 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import mimetypes
+import tempfile
+from collections import defaultdict
+from dataclasses import replace
+from math import inf
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, overload
+
+from numpy import exp, ndarray
+
+from haystack import logging
+
+if TYPE_CHECKING:
+    from haystack.dataclasses import Document
+
+CUSTOM_MIMETYPES = {
+    # we add markdown because it is not added by the mimetypes module
+    # see https://github.com/python/cpython/pull/17995
+    ".md": "text/markdown",
+    ".markdown": "text/markdown",
+    # we add msg because it is not added by the mimetypes module
+    ".msg": "application/vnd.ms-outlook",
+}
+
+logger = logging.getLogger(__name__)
+
+
+def expand_page_range(page_range: list[str | int]) -> list[int]:
+    """
+    Takes a list of page numbers and ranges and expands them into a list of page numbers.
+
+    For example, given a page_range=['1-3', '5', '8', '10-12'] the function will return [1, 2, 3, 5, 8, 10, 11, 12]
+
+    :param page_range: List of page numbers and ranges
+    :returns:
+        An expanded list of page integers
+    :raises ValueError:
+        If any element is not a valid integer or a range string in the format `'start-end'`.
+
+    """
+    expanded_page_range = []
+
+    for page in page_range:
+        if isinstance(page, int):
+            # check if it's a range wrongly passed as an integer expression
+            if "-" in str(page):
+                msg = "range must be a string in the format 'start-end'"
+                raise ValueError(f"Invalid page range: {page} - {msg}")
+            expanded_page_range.append(page)
+
+        elif isinstance(page, str) and page.isdigit():
+            expanded_page_range.append(int(page))
+
+        elif isinstance(page, str) and "-" in page:
+            parts = page.split("-", maxsplit=1)
+            if not parts[0].isdigit() or not parts[1].isdigit():
+                msg = "range must be a string in the format 'start-end'"
+                raise ValueError(f"Invalid page range: {page} - {msg}")
+            start, end = parts
+            expanded_page_range.extend(range(int(start), int(end) + 1))
+
+        else:
+            msg = "range must be a string in the format 'start-end' or an integer"
+            raise ValueError(f"Invalid page range: {page} - {msg}")
+
+    if not expanded_page_range:
+        raise ValueError("No valid page numbers or ranges found in the input list")
+
+    return expanded_page_range
+
+
+@overload
+def expit(x: float) -> float: ...
+@overload
+def expit(x: ndarray[Any, Any]) -> ndarray[Any, Any]: ...
+def expit(x: float | ndarray[Any, Any]) -> float | ndarray[Any, Any]:
+    """
+    Compute logistic sigmoid function. Maps input values to a range between 0 and 1
+
+    :param x: input value. Can be a scalar or a numpy array.
+    """
+    return 1 / (1 + exp(-x))
+
+
+def _guess_mime_type(path: Path) -> str | None:
+    """
+    Guess the MIME type of the provided file path.
+
+    :param path: The file path to get the MIME type for.
+
+    :returns: The MIME type of the provided file path, or `None` if the MIME type cannot be determined.
+    """
+    extension = path.suffix.lower()
+    mime_type = mimetypes.guess_type(path.as_posix())[0]
+    # lookup custom mappings if the mime type is not found
+    return CUSTOM_MIMETYPES.get(extension, mime_type)
+
+
+def _get_output_dir(out_dir: str) -> str:
+    """
+    Find or create a writable directory for saving status files.
+
+    Tries in the following order:
+
+        1. ~/.haystack/{out_dir}
+        2. {tempdir}/haystack/{out_dir}
+        3. ./.haystack/{out_dir}
+
+    :raises RuntimeError: If no directory could be created.
+    :returns:
+        The path to the created directory.
+    """
+
+    candidates = [
+        Path.home() / ".haystack" / out_dir,
+        Path(tempfile.gettempdir()) / "haystack" / out_dir,
+        Path.cwd() / ".haystack" / out_dir,
+    ]
+
+    for candidate in candidates:
+        try:
+            candidate.mkdir(parents=True, exist_ok=True)
+            return str(candidate)
+        except Exception:
+            continue
+
+    raise RuntimeError(
+        f"Could not create a writable directory for output files in any of the following locations: {candidates}"
+    )
+
+
+def _deduplicate_documents(documents: list["Document"]) -> list["Document"]:
+    """
+    Deduplicate a list of documents by their id keeping the duplicate with the highest score if a score is present.
+
+    :param documents: List of documents to deduplicate.
+    :returns: List of deduplicated documents.
+    """
+    # Keep for each Document id the one with the highest score
+    highest_scoring_docs: dict[str, "Document"] = {}
+    for doc in documents:
+        score = doc.score if doc.score is not None else -inf
+        best = highest_scoring_docs.get(doc.id)
+
+        if best is None or score > (best.score if best.score is not None else -inf):
+            highest_scoring_docs[doc.id] = doc
+
+    return list(highest_scoring_docs.values())
+
+
+def _reciprocal_rank_fusion(
+    document_lists: list[list["Document"]], weights: list[float] | None = None
+) -> list["Document"]:
+    """
+    Merge multiple ranked lists of Documents using Reciprocal Rank Fusion, deduplicating across lists.
+
+    See the original paper: https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf
+
+    The constant k is set to 61 (60 was suggested by the original paper, plus 1 as python lists are 0-based and the
+    paper used 1-based ranking).
+
+    :param document_lists: A list of ranked document lists to fuse.
+    :param weights: Optional per-list weights. Defaults to equal weights.
+    :returns:
+        Deduplicated list of documents with updated RRF scores.
+    """
+    if not document_lists:
+        return []
+
+    k = 61
+    scores_map: dict = defaultdict(int)
+    documents_map: dict = {}
+    resolved_weights = weights if weights else [1 / len(document_lists)] * len(document_lists)
+
+    for documents, weight in zip(document_lists, resolved_weights, strict=True):
+        for rank, doc in enumerate(documents):
+            scores_map[doc.id] += (weight * len(document_lists)) / (k + rank)
+            documents_map[doc.id] = doc
+
+    for _id in scores_map:
+        scores_map[_id] /= len(document_lists) / k
+
+    return [replace(doc, score=scores_map[doc.id]) for doc in documents_map.values()]
+
+
+@overload
+def _parse_dict_from_json(
+    text: str, expected_keys: list[str] | None = ..., raise_on_failure: Literal[True] = ...
+) -> dict[str, Any]: ...
+@overload
+def _parse_dict_from_json(
+    text: str, expected_keys: list[str] | None = ..., raise_on_failure: Literal[False] = ...
+) -> dict[str, Any] | None: ...
+@overload
+def _parse_dict_from_json(
+    text: str, expected_keys: list[str] | None = ..., raise_on_failure: bool = ...
+) -> dict[str, Any] | None: ...
+def _parse_dict_from_json(
+    text: str, expected_keys: list[str] | None = None, raise_on_failure: bool = True
+) -> dict[str, Any] | None:
+    """
+    Parses a JSON string containing a dictionary.
+
+    :param text: The string to parse.
+    :param expected_keys: A list of keys that must be present in the parsed dictionary.
+    :param raise_on_failure: If True, raises an exception on failure. If False, logs a warning and returns None.
+
+    :return: The parsed dictionary, or None if parsing fails and raise_on_failure is False.
+    :raises json.JSONDecodeError: If the text is not valid JSON and raise_on_failure is True.
+    :raises ValueError: If the parsed object is not a dictionary or has missing expected keys,
+        and `raise_on_failure` is True.
+    """
+    cleaned_text = text.strip()
+
+    try:
+        parsed_json = json.loads(cleaned_text)
+    except json.JSONDecodeError as e:
+        if raise_on_failure:
+            raise e
+        logger.warning("Failed to parse JSON from text: {text}. Error: {error}", text=text, error=e)
+        return None
+
+    if not isinstance(parsed_json, dict):
+        if raise_on_failure:
+            raise ValueError(f"Expected a JSON object containing a dictionary but got {type(parsed_json).__name__}")
+        logger.warning(
+            "Expected a JSON object containing a dictionary but got {type}. Returning None",
+            type=type(parsed_json).__name__,
+        )
+        return None
+
+    if not expected_keys:
+        return parsed_json
+
+    missing_keys = [key for key in expected_keys if key not in parsed_json]
+    if missing_keys:
+        if raise_on_failure:
+            raise ValueError(f"Missing expected keys in JSON: {missing_keys}. Got keys: {list(parsed_json.keys())}")
+        logger.warning(
+            "Missing expected keys in JSON: {missing_keys}. Got keys: {keys}",
+            missing_keys=missing_keys,
+            keys=list(parsed_json.keys()),
+        )
+        return None
+
+    return parsed_json
+
+
+def _normalize_metadata_field_name(metadata_field: str) -> str:
+    """
+    Normalizes a metadata field name by removing the "meta." prefix if present.
+    """
+    return metadata_field[5:] if metadata_field.startswith("meta.") else metadata_field

haystack/utils/requests_utils.py

@@ -0,0 +1,197 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from typing import Any
+
+import httpx
+from tenacity import after_log, before_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+# NOTE: this uses the standard library logger (not `haystack.logging`) on purpose: tenacity's `before_log`/`after_log`
+# call the logger with positional arguments, which Haystack's keyword-only patched logger would reject. We still name
+# it with `__name__` so it lives under the `haystack` namespace and is picked up by `configure_logging`.
+logger = logging.getLogger(__name__)
+
+
+def request_with_retry(
+    attempts: int = 3, status_codes_to_retry: list[int] | None = None, **kwargs: Any
+) -> httpx.Response:
+    """
+    Executes an HTTP request with a configurable exponential backoff retry on failures.
+
+    Usage example:
+    <!-- test-ignore -->
+    ```python
+    from haystack.utils import request_with_retry
+
+    # Sending an HTTP request with default retry configs
+    res = request_with_retry(method="GET", url="https://example.com")
+
+    # Sending an HTTP request with custom number of attempts
+    res = request_with_retry(method="GET", url="https://example.com", attempts=10)
+
+    # Sending an HTTP request with custom HTTP codes to retry
+    res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=[408, 503])
+
+    # Sending an HTTP request with custom timeout in seconds
+    res = request_with_retry(method="GET", url="https://example.com", timeout=5)
+
+    # Sending an HTTP request with custom headers
+    res = request_with_retry(method="GET", url="https://example.com", headers={"Authorization": "Bearer <token>"})
+
+    # Sending a POST request
+    res = request_with_retry(method="POST", url="https://example.com", json={"key": "value"}, attempts=10)
+
+    # Retry all 5xx status codes
+    res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=list(range(500, 600)))
+    ```
+
+    :param attempts:
+        Maximum number of attempts to retry the request.
+    :param status_codes_to_retry:
+        List of HTTP status codes that will trigger a retry.
+        When param is `None`, HTTP 408, 418, 429 and 503 will be retried.
+    :param kwargs:
+        Optional arguments that `httpx.Client.request` accepts.
+    :returns:
+        The `httpx.Response` object.
+    """
+
+    if status_codes_to_retry is None:
+        status_codes_to_retry = [408, 418, 429, 503]
+
+    @retry(
+        reraise=True,
+        wait=wait_exponential(),
+        retry=retry_if_exception_type((httpx.HTTPError, TimeoutError)),
+        stop=stop_after_attempt(attempts),
+        before=before_log(logger, logging.DEBUG),
+        after=after_log(logger, logging.DEBUG),
+    )
+    def run() -> httpx.Response:
+        timeout = kwargs.pop("timeout", 10)
+        with httpx.Client() as client:
+            res = client.request(**kwargs, timeout=timeout)
+
+            if res.status_code in status_codes_to_retry:
+                # We raise only for the status codes that must trigger a retry
+                res.raise_for_status()
+
+            return res
+
+    res = run()
+    # We raise here too in case the request failed with a status code that
+    # won't trigger a retry, this way the call will still cause an explicit exception
+    res.raise_for_status()
+    return res
+
+
+async def async_request_with_retry(
+    attempts: int = 3, status_codes_to_retry: list[int] | None = None, **kwargs: Any
+) -> httpx.Response:
+    """
+    Executes an asynchronous HTTP request with a configurable exponential backoff retry on failures.
+
+    Usage example:
+    ```python
+    import asyncio
+    from haystack.utils import async_request_with_retry
+
+    # Sending an async HTTP request with default retry configs
+    async def example():
+        res = await async_request_with_retry(method="GET", url="https://example.com")
+        return res
+
+    # Sending an async HTTP request with custom number of attempts
+    async def example_with_attempts():
+        res = await async_request_with_retry(method="GET", url="https://example.com", attempts=10)
+        return res
+
+    # Sending an async HTTP request with custom HTTP codes to retry
+    async def example_with_status_codes():
+        res = await async_request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=[408, 503])
+        return res
+
+    # Sending an async HTTP request with custom timeout in seconds
+    async def example_with_timeout():
+        res = await async_request_with_retry(method="GET", url="https://example.com", timeout=5)
+        return res
+
+    # Sending an async HTTP request with custom headers
+    async def example_with_headers():
+        headers = {"Authorization": "Bearer <my_token_here>"}
+        res = await async_request_with_retry(method="GET", url="https://example.com", headers=headers)
+        return res
+
+    # All of the above combined
+    async def example_combined():
+        headers = {"Authorization": "Bearer <my_token_here>"}
+        res = await async_request_with_retry(
+            method="GET",
+            url="https://example.com",
+            headers=headers,
+            attempts=10,
+            status_codes_to_retry=[408, 503],
+            timeout=5
+        )
+        return res
+
+    # Sending an async POST request
+    async def example_post():
+        res = await async_request_with_retry(
+            method="POST",
+            url="https://example.com",
+            json={"key": "value"},
+            attempts=10
+        )
+        return res
+
+    # Retry all 5xx status codes
+    async def example_5xx():
+        res = await async_request_with_retry(
+            method="GET",
+            url="https://example.com",
+            status_codes_to_retry=list(range(500, 600))
+        )
+        return res
+    ```
+
+    :param attempts:
+        Maximum number of attempts to retry the request.
+    :param status_codes_to_retry:
+        List of HTTP status codes that will trigger a retry.
+        When param is `None`, HTTP 408, 418, 429 and 503 will be retried.
+    :param kwargs:
+        Optional arguments that `httpx.AsyncClient.request` accepts.
+    :returns:
+        The `httpx.Response` object.
+    """
+
+    if status_codes_to_retry is None:
+        status_codes_to_retry = [408, 418, 429, 503]
+
+    @retry(
+        reraise=True,
+        wait=wait_exponential(),
+        retry=retry_if_exception_type((httpx.HTTPError, TimeoutError)),
+        stop=stop_after_attempt(attempts),
+        before=before_log(logger, logging.DEBUG),
+        after=after_log(logger, logging.DEBUG),
+    )
+    async def run() -> httpx.Response:
+        timeout = kwargs.pop("timeout", 10)
+        async with httpx.AsyncClient() as client:
+            res = await client.request(**kwargs, timeout=timeout)
+
+            if res.status_code in status_codes_to_retry:
+                # We raise only for the status codes that must trigger a retry
+                res.raise_for_status()
+
+            return res
+
+    res = await run()
+    # We raise here too in case the request failed with a status code that
+    # won't trigger a retry, this way the call will still cause an explicit exception
+    res.raise_for_status()
+    return res

haystack/utils/type_serialization.py

@@ -0,0 +1,264 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import builtins
+import importlib
+import inspect
+import typing
+from threading import Lock
+from types import GenericAlias, ModuleType, NoneType, UnionType
+from typing import Any, Union, get_args
+
+from haystack import logging
+from haystack.core.errors import DeserializationError
+from haystack.core.serialization_security import _check_builtin_is_type, _check_module_allowed
+
+logger = logging.getLogger(__name__)
+
+_import_lock = Lock()
+
+
+def _is_union_type(target: Any) -> bool:
+    """
+    Check if target is a Union type.
+
+    This handles both `typing.Union[X, Y]` and `X | Y` syntax from PEP 604,
+    including parameterized types like `Optional[str]`.
+    """
+    if target is Union or target is UnionType:
+        return True
+    origin = typing.get_origin(target)
+    return origin is Union or origin is UnionType
+
+
+def _build_pep604_union_type(types: list[type | UnionType]) -> type | UnionType:
+    """Build a union type from a list of types using PEP 604 syntax (X | Y)."""
+    result = types[0]
+    for t in types[1:]:
+        result = result | t
+    return result
+
+
+def serialize_type(target: Any) -> str:
+    """
+    Serializes a type or an instance to its string representation, including the module name.
+
+    This function handles types, instances of types, and special typing objects.
+    It assumes that non-typing objects will have a '__name__' attribute.
+
+    :param target:
+        The object to serialize, can be an instance or a type.
+    :return:
+        The string representation of the type.
+    """
+    if target is NoneType:
+        return "None"
+
+    args = get_args(target)
+
+    if isinstance(target, UnionType):
+        return " | ".join([serialize_type(a) for a in args])
+
+    name = getattr(target, "__name__", str(target))
+    if name.startswith("typing."):
+        name = name[7:]
+    if "[" in name:
+        name = name.split("[")[0]
+
+    # Get module name
+    module = inspect.getmodule(target)
+    module_name = ""
+    # We omit the module name for builtins to not clutter the output
+    if module and hasattr(module, "__name__") and module.__name__ != "builtins":
+        module_name = f"{module.__name__}"
+
+    if args:
+        # For typing generics, convert PEP 604 union types (X | Y) to typing.Union when serializing.
+        # This avoids issues with Python's internal cache, where List[Union[str, int]] and List[str | int] are treated
+        # as the same key. GenericAlias (builtins like list[...]) can keep the PEP 604 syntax.
+        is_typing_generic = not isinstance(target, GenericAlias)
+        # Optional[X] is normalized by Python to Union[X, None]; the trailing None is already implied by the
+        # "Optional" name, so we drop it. For any other generic (e.g. Dict[str, None], Tuple[int, None] or a
+        # Union with more than two members) NoneType is a regular argument and must be kept.
+        skip_nonetype = name == "Optional"
+        args_str = ", ".join(
+            serialize_type(Union[tuple(get_args(a))] if is_typing_generic and isinstance(a, UnionType) else a)  # noqa: UP007
+            for a in args
+            if not (skip_nonetype and a is NoneType)
+        )
+        return f"{module_name}.{name}[{args_str}]" if module_name else f"{name}[{args_str}]"
+
+    return f"{module_name}.{name}" if module_name else f"{name}"
+
+
+def _parse_generic_args(args_str: str) -> list[str]:
+    args = []
+    bracket_count = 0
+    current_arg = ""
+
+    for char in args_str:
+        if char == "[":
+            bracket_count += 1
+        elif char == "]":
+            bracket_count -= 1
+
+        if char == "," and bracket_count == 0:
+            args.append(current_arg.strip())
+            current_arg = ""
+        else:
+            current_arg += char
+
+    if current_arg:
+        args.append(current_arg.strip())
+
+    return args
+
+
+def _parse_pep604_union_args(union_str: str) -> list[str]:
+    """
+    Parse a PEP 604 union string (e.g., "str | int | None") into individual type strings.
+
+    Handles nested generics properly, e.g., "list[str] | dict[str, int] | None".
+
+    :param union_str: The union string to parse
+    :returns: A list of individual type strings
+    """
+    args = []
+    bracket_count = 0
+    current_arg = ""
+
+    for char in union_str:
+        if char == "[":
+            bracket_count += 1
+        elif char == "]":
+            bracket_count -= 1
+
+        if char == "|" and bracket_count == 0:
+            args.append(current_arg.strip())
+            current_arg = ""
+        else:
+            current_arg += char
+
+    if current_arg.strip():
+        args.append(current_arg.strip())
+
+    return args
+
+
+def deserialize_type(type_str: str) -> Any:
+    """
+    Deserializes a type given its full import path as a string, including nested generic types.
+
+    This function will dynamically import the module if it's not already imported
+    and then retrieve the type object from it. It also handles nested generic types like
+    `list[dict[int, str]]`.
+
+    Every module path with a `.` prefix is checked against the deserialization
+    allowlist (see `haystack.core.serialization_security`) before being imported. Modules outside
+    the allowlist are rejected with a `DeserializationError`. Builtin and `typing`/`collections`
+    names without a module prefix bypass this check.
+
+    :param type_str:
+        The string representation of the type's full import path.
+    :returns:
+        The deserialized type object.
+    :raises DeserializationError:
+        If the module is not on the deserialization allowlist, or if the type cannot be
+        deserialized due to a missing module or type.
+    """
+    # Handle PEP 604 union syntax at the top level (e.g., "str | int", "str | None")
+    pep604_union_args = _parse_pep604_union_args(type_str)
+    if len(pep604_union_args) > 1:
+        deserialized_args = [deserialize_type(arg) for arg in pep604_union_args]
+        return _build_pep604_union_type(deserialized_args)
+
+    # Handle generics (including Union[X, Y])
+    if "[" in type_str and type_str.endswith("]"):
+        main_type_str, generics_str = type_str.split("[", 1)
+        generics_str = generics_str[:-1]
+
+        main_type = deserialize_type(main_type_str)
+        generic_args = [deserialize_type(arg) for arg in _parse_generic_args(generics_str)]
+
+        # Reconstruct
+        try:
+            return main_type[tuple(generic_args) if len(generic_args) > 1 else generic_args[0]]
+        except (TypeError, AttributeError) as e:
+            raise DeserializationError(f"Could not apply arguments {generic_args} to type {main_type}") from e
+
+    # Handle non-generic types
+    # First, check if there's a module prefix
+    if "." in type_str:
+        try:
+            return _import_class_by_name(type_str)
+        except ImportError as e:
+            raise DeserializationError(str(e)) from e
+
+    # No module prefix, check builtins and typing
+    # Special cases for None / NoneType first: `getattr(builtins, "None")` returns the `None`
+    # singleton (not a type), so these must be handled before the builtins type gate below.
+    if type_str == "None":
+        return None
+    if type_str == "NoneType":
+        return NoneType
+
+    # Then check builtins
+    if hasattr(builtins, type_str):
+        resolved = getattr(builtins, type_str)
+        # This bare-name path never consults the allowlist. A type annotation must resolve to an
+        # actual type, so builtin functions like `eval`/`exec` are rejected while types pass.
+        _check_builtin_is_type(resolved, type_str)
+        return resolved
+
+    # Then check typing
+    if hasattr(typing, type_str):
+        return getattr(typing, type_str)
+
+    raise DeserializationError(f"Could not deserialize type: {type_str}")
+
+
+def thread_safe_import(module_name: str) -> ModuleType:
+    """
+    Import a module in a thread-safe manner.
+
+    Importing modules in a multi-threaded environment can lead to race conditions.
+    This function ensures that the module is imported in a thread-safe manner without having impact
+    on the performance of the import for single-threaded environments.
+
+    :param module_name: the module to import
+    """
+    with _import_lock:
+        return importlib.import_module(module_name)
+
+
+def _import_class_by_name(fully_qualified_name: str) -> Any:
+    """
+    Imports an attribute (typically a class) given its fully qualified name.
+
+    Checks the module against the deserialization allowlist (see
+    `haystack.core.serialization_security`) and, for `builtins`, that the resolved attribute is
+    an actual type — rejecting dangerous builtins like `eval`/`compile`.
+
+    :param fully_qualified_name: the fully qualified name, e.g. "my_package.MyClass"
+    :returns: the imported attribute.
+    :raises ImportError: If the module cannot be imported or the attribute doesn't exist on it.
+    :raises DeserializationError: If the module is not on the deserialization allowlist, or the
+        resolved builtin is not a type.
+    """
+    module_path, attr_name = fully_qualified_name.rsplit(".", 1)
+    _check_module_allowed(module_path)
+    try:
+        logger.debug(
+            "Attempting to import '{attr_name}' from module '{module_path}'",
+            attr_name=attr_name,
+            module_path=module_path,
+        )
+        module = thread_safe_import(module_path)
+        resolved = getattr(module, attr_name)
+        if module_path == "builtins":
+            _check_builtin_is_type(resolved, fully_qualified_name)
+        return resolved
+    except (ImportError, AttributeError) as error:
+        logger.exception("Failed to import '{full_name}'", full_name=fully_qualified_name)
+        raise ImportError(f"Could not import '{fully_qualified_name}'") from error

haystack/utils/url_validation.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from urllib.parse import urlparse
+
+
+def is_valid_http_url(url: str) -> bool:
+    """Check if a URL is a valid HTTP/HTTPS URL."""
+    r = urlparse(url)
+    return all([r.scheme in ["http", "https"], r.netloc])