chore: import upstream snapshot with attribution

haystack/core/pipeline/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .pipeline import Pipeline
+
+__all__ = ["Pipeline"]

haystack/core/pipeline/breakpoint.py

@@ -0,0 +1,345 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+from collections.abc import Callable
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from networkx import MultiDiGraph
+
+from haystack import logging
+from haystack.core.errors import PipelineInvalidPipelineSnapshotError
+from haystack.core.pipeline.utils import _deepcopy_with_exceptions
+from haystack.dataclasses.breakpoints import Breakpoint, PipelineSnapshot, PipelineState
+from haystack.utils.base_serialization import _serialize_value_with_schema
+
+logger = logging.getLogger(__name__)
+
+# Environment variable to control pipeline snapshot file saving (enabled by default)
+HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED = "HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED"
+
+# Type alias for snapshot callback function
+# The callback receives a PipelineSnapshot and optionally returns a file path string
+SnapshotCallback = Callable[[PipelineSnapshot], str | None]
+
+
+def _is_snapshot_save_enabled() -> bool:
+    """
+    Check if pipeline snapshot file saving is enabled via environment variable.
+
+    The environment variable HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED controls whether
+    pipeline snapshots are saved to files. By default (when the variable is not set),
+    saving is disabled. Only "true" and "1" (case-insensitive) enable saving; any other value disables it.
+
+    :returns: True if snapshot saving is enabled, False otherwise.
+    """
+    value = os.environ.get(HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED, "false").lower()
+    return value in ("true", "1")
+
+
+def _validate_break_point_against_pipeline(break_point: Breakpoint, graph: MultiDiGraph) -> None:
+    """
+    Validates the breakpoints passed to the pipeline.
+
+    Makes sure the breakpoint contains a valid components registered in the pipeline.
+
+    :param break_point: a breakpoint to validate
+    """
+    if break_point.component_name not in graph.nodes:
+        raise ValueError(f"break_point {break_point} is not a registered component in the pipeline")
+
+
+def _validate_pipeline_snapshot_against_pipeline(pipeline_snapshot: PipelineSnapshot, graph: MultiDiGraph) -> None:
+    """
+    Validates that the pipeline_snapshot contains valid configuration for the current pipeline.
+
+    Raises a PipelineInvalidPipelineSnapshotError if any component in pipeline_snapshot is not part of the
+    target pipeline.
+
+    :param pipeline_snapshot: The saved state to validate.
+    """
+
+    pipeline_state = pipeline_snapshot.pipeline_state
+    valid_components = set(graph.nodes.keys())
+
+    # Check if the ordered_component_names are valid components in the pipeline
+    invalid_ordered_components = set(pipeline_snapshot.ordered_component_names) - valid_components
+    if invalid_ordered_components:
+        raise PipelineInvalidPipelineSnapshotError(
+            f"Invalid pipeline snapshot: components {invalid_ordered_components} in 'ordered_component_names' "
+            f"are not part of the current pipeline."
+        )
+
+    # Check if the original_input_data is valid components in the pipeline
+    serialized_input_data = pipeline_snapshot.original_input_data["serialized_data"]
+    invalid_input_data = set(serialized_input_data.keys()) - valid_components
+    if invalid_input_data:
+        raise PipelineInvalidPipelineSnapshotError(
+            f"Invalid pipeline snapshot: components {invalid_input_data} in 'input_data' "
+            f"are not part of the current pipeline."
+        )
+
+    # Validate 'component_visits'
+    invalid_component_visits = set(pipeline_state.component_visits.keys()) - valid_components
+    if invalid_component_visits:
+        raise PipelineInvalidPipelineSnapshotError(
+            f"Invalid pipeline snapshot: components {invalid_component_visits} in 'component_visits' "
+            f"are not part of the current pipeline."
+        )
+
+    component_name = pipeline_snapshot.break_point.component_name
+    visit_count = pipeline_snapshot.pipeline_state.component_visits[component_name]
+
+    logger.info(
+        "Resuming pipeline from {component} with visit count {visits}", component=component_name, visits=visit_count
+    )
+
+
+def load_pipeline_snapshot(file_path: str | Path) -> PipelineSnapshot:
+    """
+    Load a saved pipeline snapshot.
+
+    :param file_path: Path to the pipeline_snapshot file.
+    :returns:
+        Dict containing the loaded pipeline_snapshot.
+    """
+
+    file_path = Path(file_path)
+
+    try:
+        with open(file_path, encoding="utf-8") as f:
+            pipeline_snapshot_dict = json.load(f)
+    except FileNotFoundError as e:
+        raise FileNotFoundError(f"File not found: {file_path}") from e
+    except json.JSONDecodeError as e:
+        raise json.JSONDecodeError(f"Invalid JSON file {file_path}: {str(e)}", e.doc, e.pos) from e
+    except OSError as e:
+        raise OSError(f"Error reading {file_path}: {str(e)}") from e
+
+    try:
+        pipeline_snapshot = PipelineSnapshot.from_dict(pipeline_snapshot_dict)
+    except ValueError as e:
+        raise ValueError(f"Invalid pipeline snapshot from {file_path}: {str(e)}") from e
+
+    logger.info("Successfully loaded the pipeline snapshot from: {file_path}", file_path=file_path)
+    return pipeline_snapshot
+
+
+def _save_pipeline_snapshot(
+    pipeline_snapshot: PipelineSnapshot,
+    raise_on_failure: bool = True,
+    snapshot_callback: SnapshotCallback | None = None,
+) -> str | None:
+    """
+    Save the pipeline snapshot dictionary to a JSON file, or invoke a custom callback.
+
+    If a `snapshot_callback` is provided, it will be called with the pipeline snapshot instead of saving to a file.
+    This allows users to customize how snapshots are handled (e.g., saving to a database, sending to a remote service).
+
+    When no callback is provided, the default behavior saves to a JSON file:
+    - The filename is generated based on the component name, visit count, and timestamp.
+        - The component name is taken from the break point's `component_name`.
+        - The visit count is taken from the pipeline state's `component_visits` for the component name.
+        - The timestamp is taken from the pipeline snapshot's `timestamp` or the current time if not available.
+    - The file path is taken from the break point's `snapshot_file_path`.
+    - If the `snapshot_file_path` is None, the function will return without saving.
+
+    The default file saving behavior is disabled. To enable it, set the environment variable
+    `HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED` to "true" or "1". When disabled,
+    the function will return None without saving to a file (custom callbacks are still invoked).
+
+    :param pipeline_snapshot: The pipeline snapshot to save.
+    :param raise_on_failure: If True, raises an exception if saving fails. If False, logs the error and returns.
+    :param snapshot_callback: Optional callback function that receives the PipelineSnapshot.
+        If provided, the callback is invoked instead of the default file-saving behavior.
+        The callback should return an optional string (e.g., a file path or identifier) or None.
+
+    :returns:
+        The full path to the saved JSON file (or the value returned by the callback), or None if
+        `snapshot_file_path` is None, no callback is provided, or snapshot saving is disabled.
+    :raises:
+        Exception: If saving the JSON snapshot fails (when raise_on_failure is True).
+    """
+    # If a callback is provided, use it instead of the default file-saving behavior
+    if snapshot_callback is not None:
+        try:
+            result = snapshot_callback(pipeline_snapshot)
+            logger.info("Pipeline snapshot handled by custom callback.")
+            return result
+        except Exception as error:
+            logger.exception("Failed to handle pipeline snapshot with custom callback. Error: {error}", error=error)
+            if raise_on_failure:
+                raise
+            return None
+
+    # Check if snapshot saving is enabled via environment variable (enabled by default)
+    if not _is_snapshot_save_enabled():
+        logger.debug("Pipeline snapshot file saving is disabled via HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED env var.")
+        return None
+
+    break_point = pipeline_snapshot.break_point
+    snapshot_file_path = break_point.snapshot_file_path
+
+    if snapshot_file_path is None:
+        return None
+
+    dt = pipeline_snapshot.timestamp or datetime.now()
+    snapshot_dir = Path(snapshot_file_path)
+
+    component_name = break_point.component_name
+    visit_nr = pipeline_snapshot.pipeline_state.component_visits.get(component_name, 0)
+    timestamp = dt.strftime("%Y_%m_%d_%H_%M_%S")
+    file_name = f"{component_name}_{visit_nr}_{timestamp}.json"
+    full_path = snapshot_dir / file_name
+
+    try:
+        snapshot_dir.mkdir(parents=True, exist_ok=True)
+        with open(full_path, "w") as f_out:
+            json.dump(pipeline_snapshot.to_dict(), f_out, indent=2)
+        logger.info(
+            "Pipeline snapshot saved to '{full_path}'. You can use this file to debug or resume the pipeline.",
+            full_path=full_path,
+        )
+    except Exception as error:
+        logger.exception("Failed to save pipeline snapshot to '{full_path}'. Error: {e}", full_path=full_path, e=error)
+        if raise_on_failure:
+            raise
+
+    return str(full_path)
+
+
+def _create_pipeline_snapshot(
+    *,
+    inputs: dict[str, Any],
+    component_inputs: dict[str, Any],
+    break_point: Breakpoint,
+    component_visits: dict[str, int],
+    original_input_data: dict[str, Any],
+    ordered_component_names: list[str],
+    include_outputs_from: set[str],
+    pipeline_outputs: dict[str, Any],
+) -> PipelineSnapshot:
+    """
+    Create a snapshot of the pipeline at the point where the breakpoint was triggered.
+
+    :param inputs: The current pipeline snapshot inputs.
+    :param component_inputs: The inputs to the component that triggered the breakpoint.
+    :param break_point: The breakpoint that triggered the snapshot.
+    :param component_visits: The visit count of the component that triggered the breakpoint.
+    :param original_input_data: The original input data.
+    :param ordered_component_names: The ordered component names.
+    :param include_outputs_from: Set of component names whose outputs should be included in the pipeline results.
+    :param pipeline_outputs: The current outputs of the pipeline.
+    :returns:
+        A PipelineSnapshot containing the state of the pipeline at the point of the breakpoint.
+    """
+    component_name = break_point.component_name
+
+    transformed_original_input_data = _transform_json_structure(original_input_data)
+    transformed_inputs = _transform_json_structure({**inputs, component_name: component_inputs})
+
+    serialized_inputs = _serialize_with_field_fallback(
+        transformed_inputs, description="the inputs of the current pipeline state"
+    )
+    serialized_original_input_data = _serialize_with_field_fallback(
+        transformed_original_input_data, description="original input data for `pipeline.run`"
+    )
+    serialized_pipeline_outputs = _serialize_with_field_fallback(
+        pipeline_outputs, description="outputs of the current pipeline state"
+    )
+
+    return PipelineSnapshot(
+        pipeline_state=PipelineState(
+            inputs=serialized_inputs, component_visits=component_visits, pipeline_outputs=serialized_pipeline_outputs
+        ),
+        timestamp=datetime.now(),
+        break_point=break_point,
+        original_input_data=serialized_original_input_data,
+        ordered_component_names=ordered_component_names,
+        include_outputs_from=include_outputs_from,
+    )
+
+
+def _transform_json_structure(data: dict[str, Any] | list[Any] | Any) -> Any:
+    """
+    Transforms a JSON structure by removing the 'sender' key and moving the 'value' to the top level.
+
+    For example:
+    "key": [{"sender": null, "value": "some value"}] -> "key": "some value"
+
+    :param data: The JSON structure to transform.
+    :returns: The transformed structure.
+    """
+    if isinstance(data, dict):
+        # If this dict has both 'sender' and 'value', return just the value
+        if "value" in data and "sender" in data:
+            return data["value"]
+        # Otherwise, recursively process each key-value pair
+        return {k: _transform_json_structure(v) for k, v in data.items()}
+
+    if isinstance(data, list):
+        # First, transform each item in the list.
+        transformed = [_transform_json_structure(item) for item in data]
+        # If the original list has exactly one element and that element was a dict
+        # with 'sender' and 'value', then unwrap the list.
+        if len(data) == 1 and isinstance(data[0], dict) and "value" in data[0] and "sender" in data[0]:
+            return transformed[0]
+        return transformed
+
+    # For other data types, just return the value as is.
+    return data
+
+
+def _serialize_with_field_fallback(payload: Any, *, description: str) -> dict[str, Any]:
+    """
+    Serialize a payload and, on failure, retry field-by-field to preserve resumable fields.
+
+    If the whole payload serializes, the result is returned as-is. Otherwise, and if the payload is a
+    mapping, each top-level field is serialized individually and only the failing fields are omitted.
+    When the payload is not a mapping, or when every field fails to serialize, the helper returns a
+    structurally valid empty-object payload so that the downstream ``_deserialize_value_with_schema``
+    can still load it back instead of raising ``DeserializationError`` on a bare ``{}``.
+
+    :param payload: The value to serialize.
+    :param description: Short human-readable label used in warning messages, for example
+        ``"the agent's chat_generator inputs"`` or ``"the inputs of the current pipeline state"``.
+    :returns: A dict of the form ``{"serialization_schema": ..., "serialized_data": ...}``.
+    """
+    try:
+        return _serialize_value_with_schema(_deepcopy_with_exceptions(payload))
+    except Exception as error:
+        logger.warning(
+            "Failed to serialize {description}. "
+            "Haystack will omit only the non-serializable fields when possible. Error: {e}",
+            description=description,
+            e=error,
+        )
+
+    serialized_properties: dict[str, Any] = {}
+    serialized_data: dict[str, Any] = {}
+
+    if isinstance(payload, dict):
+        for field_name, value in payload.items():
+            try:
+                serialized_value = _serialize_value_with_schema(_deepcopy_with_exceptions(value))
+            except Exception as field_error:
+                logger.warning(
+                    "Failed to serialize the '{field_name}' field of {description}. "
+                    "The field will be omitted from the snapshot. Error: {e}",
+                    field_name=field_name,
+                    description=description,
+                    e=field_error,
+                )
+                continue
+
+            serialized_properties[field_name] = serialized_value["serialization_schema"]
+            serialized_data[field_name] = serialized_value["serialized_data"]
+
+    return {
+        "serialization_schema": {"type": "object", "properties": serialized_properties},
+        "serialized_data": serialized_data,
+    }

haystack/core/pipeline/component_checks.py

@@ -0,0 +1,215 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack.core.component.types import InputSocket, _empty
+
+_NO_OUTPUT_PRODUCED = _empty
+
+
+def can_component_run(component: dict, inputs: dict) -> bool:
+    """
+    Checks if the component can run, given the current state of its inputs.
+
+    A component needs to pass two gates so that it is ready to run:
+    1. It has received all mandatory inputs.
+    2. It has received a trigger.
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    """
+    received_all_mandatory_inputs = are_all_sockets_ready(component, inputs, only_check_mandatory=True)
+    received_trigger = has_any_trigger(component, inputs)
+
+    return received_all_mandatory_inputs and received_trigger
+
+
+def has_any_trigger(component: dict, inputs: dict) -> bool:
+    """
+    Checks if a component was triggered to execute.
+
+    There are 3 triggers:
+    1. A predecessor provided input to the component.
+    2. Input to the component was provided from outside the pipeline (e.g. user input).
+    3. The component does not receive input from any other components in the pipeline and `Pipeline.run` was called.
+
+    A trigger can only cause a component to execute ONCE because:
+    1. Components consume inputs from predecessors before execution (they are deleted).
+    2. Inputs from outside the pipeline can only trigger a component when it is executed for the first time.
+    3. `Pipeline.run` can only trigger a component when it is executed for the first time.
+
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    """
+    trigger_from_predecessor = any_predecessors_provided_input(component, inputs)
+    trigger_from_user = has_user_input(inputs) and component["visits"] == 0
+    trigger_without_inputs = can_not_receive_inputs_from_pipeline(component) and component["visits"] == 0
+
+    return trigger_from_predecessor or trigger_from_user or trigger_without_inputs
+
+
+def are_all_sockets_ready(component: dict, inputs: dict, only_check_mandatory: bool = False) -> bool:
+    """
+    Checks if all sockets of a component have enough inputs for the component to execute.
+
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    :param only_check_mandatory: If only mandatory sockets should be checked.
+    """
+    filled_sockets = set()
+    expected_sockets = set()
+    if only_check_mandatory:
+        sockets_to_check = {
+            socket_name: socket for socket_name, socket in component["input_sockets"].items() if socket.is_mandatory
+        }
+    else:
+        sockets_to_check = {
+            socket_name: socket
+            for socket_name, socket in component["input_sockets"].items()
+            if socket.is_mandatory or len(socket.senders)
+        }
+
+    for socket_name, socket in sockets_to_check.items():
+        socket_inputs = inputs.get(socket_name, [])
+        expected_sockets.add(socket_name)
+
+        # Check if socket has all required inputs or is a lazy variadic socket with any input
+        if has_socket_received_all_inputs(socket, socket_inputs) or (
+            socket.is_lazy_variadic and any_socket_input_received(socket_inputs)
+        ):
+            filled_sockets.add(socket_name)
+
+    return filled_sockets == expected_sockets
+
+
+def any_predecessors_provided_input(component: dict, inputs: dict) -> bool:
+    """
+    Checks if a component received inputs from any predecessors.
+
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    """
+    return any(
+        any_socket_value_from_predecessor_received(inputs.get(socket_name, []))
+        for socket_name in component["input_sockets"].keys()
+    )
+
+
+def any_socket_value_from_predecessor_received(socket_inputs: list[dict[str, Any]]) -> bool:
+    """
+    Checks if a component socket received input from any predecessors.
+
+    :param socket_inputs: Inputs for the component's socket.
+    """
+    # When sender is None, the input was provided from outside the pipeline.
+    return any(inp["value"] is not _NO_OUTPUT_PRODUCED and inp["sender"] is not None for inp in socket_inputs)
+
+
+def has_user_input(inputs: dict) -> bool:
+    """
+    Checks if a component has received input from outside the pipeline (e.g. user input).
+
+    :param inputs: Inputs for the component.
+    """
+    return any(inp for socket in inputs.values() for inp in socket if inp["sender"] is None)
+
+
+def can_not_receive_inputs_from_pipeline(component: dict) -> bool:
+    """
+    Checks if a component can not receive inputs from any other components in the pipeline.
+
+    :param: Component metadata and the component instance.
+    """
+    return all(len(sock.senders) == 0 for sock in component["input_sockets"].values())
+
+
+def all_socket_predecessors_executed(socket: InputSocket, socket_inputs: list[dict[str, Any]]) -> bool:
+    """
+    Checks if all components connecting to an InputSocket have executed.
+
+    :param: The InputSocket of a component.
+    :param: socket_inputs: Inputs for the socket.
+    """
+    expected_senders = set(socket.senders)
+    executed_senders = {inp["sender"] for inp in socket_inputs if inp["sender"] is not None}
+    return expected_senders == executed_senders
+
+
+def any_socket_input_received(socket_inputs: list[dict]) -> bool:
+    """
+    Checks if a socket has received any input from any other components in the pipeline or from outside the pipeline.
+
+    :param socket_inputs: Inputs for the socket.
+    """
+    return any(inp["value"] is not _NO_OUTPUT_PRODUCED for inp in socket_inputs)
+
+
+def has_lazy_variadic_socket_received_all_inputs(socket: InputSocket, socket_inputs: list[dict]) -> bool:
+    """
+    Checks if a lazy variadic socket has received all expected inputs from other components in the pipeline.
+
+    :param socket: The InputSocket of a component.
+    :param socket_inputs: Inputs for the socket.
+    """
+    expected_senders = set(socket.senders)
+    actual_senders = {
+        sock["sender"]
+        for sock in socket_inputs
+        if sock["value"] is not _NO_OUTPUT_PRODUCED and sock["sender"] is not None
+    }
+    return expected_senders == actual_senders
+
+
+def has_socket_received_all_inputs(socket: InputSocket, socket_inputs: list[dict]) -> bool:
+    """
+    Checks if a socket has received all expected inputs.
+
+    :param socket: The InputSocket of a component.
+    :param socket_inputs: Inputs for the socket.
+    """
+    # No inputs received for the socket, it is not filled.
+    if len(socket_inputs) == 0:
+        return False
+
+    # The socket is greedy variadic and at least one input was produced, it is complete.
+    if (
+        socket.is_variadic
+        and socket.is_greedy
+        and any(sock["value"] is not _NO_OUTPUT_PRODUCED for sock in socket_inputs)
+    ):
+        return True
+
+    # The socket is lazy variadic and all expected inputs were produced.
+    if socket.is_lazy_variadic and has_lazy_variadic_socket_received_all_inputs(socket, socket_inputs):
+        return True
+
+    # The socket is not variadic and the only expected input is complete.
+    return not socket.is_variadic and socket_inputs[0]["value"] is not _NO_OUTPUT_PRODUCED
+
+
+def all_predecessors_executed(component: dict, inputs: dict) -> bool:
+    """
+    Checks if all predecessors of a component have executed.
+
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    """
+    return all(
+        all_socket_predecessors_executed(socket, inputs.get(socket_name, []))
+        for socket_name, socket in component["input_sockets"].items()
+    )
+
+
+def is_any_greedy_socket_ready(component: dict, inputs: dict) -> bool:
+    """
+    Checks if the component has any greedy socket that is ready to run.
+
+    :param component: Component metadata and the component instance.
+    :param inputs: Inputs for the component.
+    """
+    for socket_name, socket in component["input_sockets"].items():
+        if socket.is_greedy and has_socket_received_all_inputs(socket, inputs.get(socket_name, [])):
+            return True
+
+    return False

haystack/core/pipeline/descriptions.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+import networkx
+
+from haystack.core.component.types import InputSocket, OutputSocket
+
+
+def find_pipeline_inputs(
+    graph: networkx.MultiDiGraph, include_connected_sockets: bool = False
+) -> dict[str, list[InputSocket]]:
+    """
+    Collect components that have disconnected/connected input sockets.
+
+    Note that this method returns *ALL* disconnected input sockets, including all such sockets with default values.
+    It also includes variadic input sockets, even if they are currently connected, as they can accept additional
+    inputs from outside the pipeline.
+
+    :param graph: The pipeline graph to analyze.
+    :param include_connected_sockets: If True, also include input sockets that are already connected.
+        This can be useful for understanding the full input requirements of the pipeline, including inputs
+        that are currently satisfied by connections within the pipeline. If False, only include input sockets that
+        are not connected to any output socket, which represent the external inputs that can be provided when running
+        the pipeline.
+    """
+    return {
+        name: [
+            socket
+            for socket in data.get("input_sockets", {}).values()
+            if socket.is_variadic or (include_connected_sockets or not socket.senders)
+        ]
+        for name, data in graph.nodes(data=True)
+    }
+
+
+def find_pipeline_outputs(
+    graph: networkx.MultiDiGraph, include_connected_sockets: bool = False
+) -> dict[str, list[OutputSocket]]:
+    """
+    Collect components that have disconnected/connected output sockets. They define the pipeline output.
+    """
+    return {
+        name: [
+            socket
+            for socket in data.get("output_sockets", {}).values()
+            if (include_connected_sockets or not socket.receivers)
+        ]
+        for name, data in graph.nodes(data=True)
+    }

haystack/core/pipeline/draw.py

@@ -0,0 +1,432 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import base64
+import colorsys
+import json
+import random
+import zlib
+from typing import Any
+
+import httpx
+import networkx
+
+from haystack import logging
+from haystack.core.errors import PipelineDrawingError
+from haystack.core.pipeline.descriptions import find_pipeline_inputs, find_pipeline_outputs
+from haystack.core.type_utils import _type_name
+
+logger = logging.getLogger(__name__)
+
+
+def generate_color_variations(n: int, base_color: str | None = "#3498DB", variation_range: float = 0.4) -> list[str]:
+    """
+    Generate n different variations of a base color.
+
+    :param n: Number of variations to generate
+    :param base_color: Hex color code, default is a shade of blue (#3498DB)
+    :param variation_range: Range for varying brightness and saturation (0-1)
+
+    :returns:
+        list: List of hex color codes representing variations of the base color
+    """
+    # convert hex to RGB
+    base_color = base_color.lstrip("#")  # type:ignore
+    r = int(base_color[0:2], 16) / 255.0
+    g = int(base_color[2:4], 16) / 255.0
+    b = int(base_color[4:6], 16) / 255.0
+
+    # convert RGB to HSV (Hue, Saturation, Value)
+    h, s, v = colorsys.rgb_to_hsv(r, g, b)
+
+    variations = []
+    for _ in range(n):
+        # vary saturation and brightness within the specified range
+        new_s = max(0, min(1, s + random.uniform(-variation_range, variation_range)))
+        new_v = max(0, min(1, v + random.uniform(-variation_range, variation_range)))
+
+        # keep hue the same for color consistency
+        new_h = h
+
+        # Convert back to RGB and then to hex
+        new_r, new_g, new_b = colorsys.hsv_to_rgb(new_h, new_s, new_v)
+        hex_color = f"#{int(new_r * 255):02x}{int(new_g * 255):02x}{int(new_b * 255):02x}"
+
+        variations.append(hex_color)
+
+    return variations
+
+
+def _prepare_for_drawing(graph: networkx.MultiDiGraph) -> networkx.MultiDiGraph:
+    """
+    Add some extra nodes to show the inputs and outputs of the pipeline.
+
+    Also adds labels to edges.
+    """
+    # Label the edges
+    for inp, outp, key, data in graph.edges(keys=True, data=True):
+        data["label"] = (
+            f"{data['from_socket'].name} -> {data['to_socket'].name}{' (opt.)' if not data['mandatory'] else ''}"
+        )
+        graph.add_edge(inp, outp, key=key, **data)
+
+    # Add inputs fake node
+    graph.add_node("input")
+    for node, in_sockets in find_pipeline_inputs(graph).items():
+        for in_socket in in_sockets:
+            if not in_socket.senders and in_socket.is_mandatory:
+                # If this socket has no sender it could be a socket that receives input
+                # directly when running the Pipeline. We can't know that for sure, in doubt
+                # we draw it as receiving input directly.
+                graph.add_edge("input", node, label=in_socket.name, conn_type=_type_name(in_socket.type))
+
+    # Add outputs fake node
+    graph.add_node("output")
+    for node, out_sockets in find_pipeline_outputs(graph).items():
+        for out_socket in out_sockets:
+            graph.add_edge(node, "output", label=out_socket.name, conn_type=_type_name(out_socket.type))
+
+    return graph
+
+
+ARROWTAIL_MANDATORY = "--"
+ARROWTAIL_OPTIONAL = "-."
+ARROWHEAD_MANDATORY = "-->"
+ARROWHEAD_OPTIONAL = ".->"
+MERMAID_STYLED_TEMPLATE = """
+%%{{ init: {params} }}%%
+
+graph TD;
+
+{connections}
+
+classDef component text-align:center;
+{style_definitions}
+"""
+
+
+def _validate_mermaid_params(params: dict[str, Any]) -> None:
+    """
+    Validates and sets default values for Mermaid parameters.
+
+    :param params:
+        Dictionary of customization parameters to modify the output. Refer to Mermaid documentation for more details.
+        Supported keys:
+            - format: Output format ('img', 'svg', or 'pdf'). Default: 'img'.
+            - type: Image type for /img endpoint ('jpeg', 'png', 'webp'). Default: 'png'.
+            - theme: Mermaid theme ('default', 'neutral', 'dark', 'forest'). Default: 'neutral'.
+            - bgColor: Background color in hexadecimal (e.g., 'FFFFFF') or named format (e.g., '!white').
+            - width: Width of the output image (integer).
+            - height: Height of the output image (integer).
+            - scale: Scaling factor (1–3). Only applicable if 'width' or 'height' is specified.
+            - fit: Whether to fit the diagram size to the page (PDF only, boolean).
+            - paper: Paper size for PDFs (e.g., 'a4', 'a3'). Ignored if 'fit' is true.
+            - landscape: Landscape orientation for PDFs (boolean). Ignored if 'fit' is true.
+
+    :raises ValueError:
+        If any parameter is invalid or does not match the expected format.
+    """
+    valid_img_types = {"jpeg", "png", "webp"}
+    valid_themes = {"default", "neutral", "dark", "forest"}
+    valid_formats = {"img", "svg", "pdf"}
+
+    params.setdefault("format", "img")
+    params.setdefault("type", "png")
+    params.setdefault("theme", "neutral")
+
+    if params["format"] not in valid_formats:
+        raise ValueError(f"Invalid image format: {params['format']}. Valid options are: {valid_formats}.")
+
+    if params["format"] == "img" and params["type"] not in valid_img_types:
+        raise ValueError(f"Invalid image type: {params['type']}. Valid options are: {valid_img_types}.")
+
+    if params["theme"] not in valid_themes:
+        raise ValueError(f"Invalid theme: {params['theme']}. Valid options are: {valid_themes}.")
+
+    if "width" in params and not isinstance(params["width"], int):
+        raise ValueError("Width must be an integer.")
+    if "height" in params and not isinstance(params["height"], int):
+        raise ValueError("Height must be an integer.")
+
+    if "scale" in params and not 1 <= params["scale"] <= 3:
+        raise ValueError("Scale must be a number between 1 and 3.")
+    if "scale" in params and not ("width" in params or "height" in params):
+        raise ValueError("Scale is only allowed when width or height is set.")
+
+    if "bgColor" in params and not isinstance(params["bgColor"], str):
+        raise ValueError("Background color must be a string.")
+
+    # PDF specific parameters
+    if params["format"] == "pdf":
+        if "fit" in params and not isinstance(params["fit"], bool):
+            raise ValueError("Fit must be a boolean.")
+        if "paper" in params and not isinstance(params["paper"], str):
+            raise ValueError("Paper size must be a string (e.g., 'a4', 'a3').")
+        if "landscape" in params and not isinstance(params["landscape"], bool):
+            raise ValueError("Landscape must be a boolean.")
+        if "fit" in params and ("paper" in params or "landscape" in params):
+            logger.warning("`fit` overrides `paper` and `landscape` for PDFs. Ignoring `paper` and `landscape`.")
+
+
+# Magic-byte signatures used to verify a Mermaid server response matches the requested output format.
+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
+_JPEG_SIGNATURE = b"\xff\xd8\xff"
+_PDF_SIGNATURE = b"%PDF-"
+_RIFF_SIGNATURE = b"RIFF"
+_WEBP_SIGNATURE = b"WEBP"
+_SVG_PREFIXES = (b"<?xml", b"<svg")
+
+
+def _validate_image_response(resp: httpx.Response, params: dict[str, Any]) -> None:
+    """
+    Validate that the Mermaid server response actually contains the expected image/SVG/PDF data.
+
+    `Pipeline.draw()` writes the raw response body to disk, so a misconfigured or malicious
+    `server_url` could otherwise cause arbitrary content (e.g. an HTML error page or a crafted
+    payload) to be written verbatim to the output path. As defense-in-depth we check both the
+    `Content-Type` header (which the server controls and could spoof) and the response body's
+    magic-byte signature (which is harder to forge while still producing a usable payload).
+
+    :param resp:
+        The HTTP response returned by the Mermaid server.
+    :param params:
+        Validated Mermaid parameters; used to determine the expected output format.
+    :raises PipelineDrawingError:
+        If the response is empty or does not match the expected format.
+    """
+    content = resp.content
+    if not content:
+        raise PipelineDrawingError("The Mermaid server returned an empty response; no image will be saved.")
+
+    output_format = params.get("format", "img")
+    img_type = params.get("type", "png")
+
+    # (human-readable label, expected Content-Type prefix, body signature check)
+    content_type_prefixes: tuple[str, ...]
+    if output_format == "svg":
+        expected_label = "SVG"
+        content_type_prefixes = ("image/svg+xml", "text/xml", "application/xml")
+        stripped = content.lstrip()[:512].lower()
+        body_ok = stripped.startswith(_SVG_PREFIXES) or _SVG_PREFIXES[1] in stripped
+    elif output_format == "pdf":
+        expected_label = "PDF"
+        content_type_prefixes = ("application/pdf",)
+        body_ok = content.startswith(_PDF_SIGNATURE)
+    elif img_type == "jpeg":
+        expected_label = "JPEG image"
+        content_type_prefixes = ("image/jpeg",)
+        body_ok = content.startswith(_JPEG_SIGNATURE)
+    elif img_type == "webp":
+        expected_label = "WebP image"
+        content_type_prefixes = ("image/webp",)
+        body_ok = content[0:4] == _RIFF_SIGNATURE and content[8:12] == _WEBP_SIGNATURE
+    else:  # png (default)
+        expected_label = "PNG image"
+        content_type_prefixes = ("image/png",)
+        body_ok = content.startswith(_PNG_SIGNATURE)
+
+    # The Content-Type header is server-controlled, so a mismatch is only a warning: the
+    # authoritative check is the body signature below.
+    content_type = resp.headers.get("content-type", "").split(";")[0].strip().lower()
+    if content_type and not content_type.startswith(content_type_prefixes):
+        logger.warning(
+            "The Mermaid server returned an unexpected Content-Type '{content_type}' (expected {expected}).",
+            content_type=content_type,
+            expected=expected_label,
+        )
+
+    if not body_ok:
+        raise PipelineDrawingError(
+            f"The Mermaid server response does not look like a valid {expected_label}. "
+            f"This can happen if 'server_url' points to a server that is not a Mermaid renderer. "
+            f"To avoid writing untrusted content to disk, no file will be saved."
+        )
+
+
+def _to_mermaid_image(
+    graph: networkx.MultiDiGraph,
+    server_url: str = "https://mermaid.ink",
+    params: dict | None = None,
+    timeout: int = 30,
+    super_component_mapping: dict[str, str] | None = None,
+) -> bytes:
+    """
+    Renders a pipeline using a Mermaid server.
+
+    :param graph:
+        The graph to render as a Mermaid pipeline.
+    :param server_url:
+        Base URL of the Mermaid server (default: 'https://mermaid.ink').
+    :param params:
+        Dictionary of customization parameters. See `validate_mermaid_params` for valid keys.
+    :param timeout:
+        Timeout in seconds for the request to the Mermaid server.
+    :returns:
+        The image, SVG, or PDF data returned by the Mermaid server as bytes.
+    :raises ValueError:
+        If any parameter is invalid or does not match the expected format.
+    :raises PipelineDrawingError:
+        If there is an issue connecting to the Mermaid server or the server returns an error.
+    """
+
+    if params is None:
+        params = {}
+
+    _validate_mermaid_params(params)
+
+    theme = params.get("theme")
+    init_params = json.dumps({"theme": theme})
+
+    # Copy the graph to avoid modifying the original
+    graph_styled = _to_mermaid_text(graph.copy(), init_params, super_component_mapping)
+    json_string = json.dumps({"code": graph_styled})
+
+    # Compress the JSON string with zlib (RFC 1950)
+    compressor = zlib.compressobj(level=9, wbits=15)
+    compressed_data = compressor.compress(json_string.encode("utf-8")) + compressor.flush()
+    compressed_url_safe_base64 = base64.urlsafe_b64encode(compressed_data).decode("utf-8").strip()
+
+    # Determine the correct endpoint
+    endpoint_format = params.get("format", "img")  # Default to /img endpoint
+    if endpoint_format not in {"img", "svg", "pdf"}:
+        raise ValueError(f"Invalid format: {endpoint_format}. Valid options are 'img', 'svg', or 'pdf'.")
+
+    # Construct the URL without query parameters
+    url = f"{server_url}/{endpoint_format}/pako:{compressed_url_safe_base64}"
+
+    # Add query parameters adhering to mermaid.ink documentation
+    query_params = []
+    for key, value in params.items():
+        if key not in {"theme", "format"}:  # Exclude theme (handled in init_params) and format (endpoint-specific)
+            if value is True:
+                query_params.append(f"{key}")
+            else:
+                query_params.append(f"{key}={value}")
+
+    if query_params:
+        url += "?" + "&".join(query_params)
+
+    logger.debug("Rendering graph at {url}", url=url)
+    try:
+        resp = httpx.get(url, timeout=timeout)
+        if resp.status_code >= 400:
+            logger.warning(
+                "Failed to draw the pipeline: {server_url} returned status {status_code}",
+                server_url=server_url,
+                status_code=resp.status_code,
+            )
+            logger.info("Exact URL requested: {url}", url=url)
+            logger.warning("No pipeline diagram will be saved.")
+            resp.raise_for_status()
+
+    except Exception as exc:
+        logger.warning(
+            "Failed to draw the pipeline: could not connect to {server_url} ({error})", server_url=server_url, error=exc
+        )
+        logger.info("Exact URL requested: {url}", url=url)
+        logger.warning("No pipeline diagram will be saved.")
+        raise PipelineDrawingError(f"There was an issue with {server_url}, see the stacktrace for details.") from exc
+
+    # Validate the response before it gets written to disk by the caller, so that a misconfigured
+    # or malicious server cannot cause arbitrary content to be saved to the output path.
+    _validate_image_response(resp, params)
+
+    return resp.content
+
+
+def _to_mermaid_text(
+    graph: networkx.MultiDiGraph, init_params: str | dict, super_component_mapping: dict[str, str] | None = None
+) -> str:
+    """
+    Converts a Networkx graph into Mermaid syntax.
+
+    The output of this function can be used in the documentation with `mermaid` codeblocks and will be
+    automatically rendered.
+
+    :param graph: The graph to convert to Mermaid syntax
+    :param init_params: Initialization parameters for Mermaid
+    :param super_component_mapping: Mapping of component names to super component names
+    """
+    # Copy the graph to avoid modifying the original
+    graph = _prepare_for_drawing(graph.copy())
+    sockets = {
+        comp: "".join(
+            [
+                f"<li>{name} ({_type_name(socket.type)})</li>"
+                for name, socket in data.get("input_sockets", {}).items()
+                if (not socket.is_mandatory and not socket.senders) or socket.is_variadic
+            ]
+        )
+        for comp, data in graph.nodes(data=True)
+    }
+    optional_inputs = {
+        comp: f"<br><br>Optional inputs:<ul style='text-align:left;'>{sockets}</ul>" if sockets else ""
+        for comp, sockets in sockets.items()
+    }
+
+    # Create node definitions
+    states = {}
+    super_component_components = super_component_mapping.keys() if super_component_mapping else {}
+
+    # color variations for super components
+    super_component_colors = {}
+    if super_component_components:
+        unique_super_components = set(super_component_mapping.values())  # type:ignore
+        color_variations = generate_color_variations(n=len(unique_super_components))
+        super_component_colors = dict(zip(unique_super_components, color_variations, strict=True))
+
+    # Generate style definitions for each super component
+    style_definitions = []
+    for super_comp, color in super_component_colors.items():
+        style_definitions.append(f"classDef {super_comp} fill:{color},color:white;")
+
+    for comp, data in graph.nodes(data=True):
+        if comp in ["input", "output"]:
+            continue
+
+        # styling based on whether the component is a SuperComponent
+        if comp in super_component_components:
+            super_component_name = super_component_mapping[comp]  # type:ignore
+            style = super_component_name
+        else:
+            style = "component"
+        node_def = f'{comp}["<b>{comp}</b><br><small><i>{type(data["instance"]).__name__}{optional_inputs[comp]}</i></small>"]:::{style}'  # noqa: E501
+        states[comp] = node_def
+
+    connections_list = []
+    for from_comp, to_comp, conn_data in graph.edges(data=True):
+        if from_comp != "input" and to_comp != "output":
+            arrowtail = ARROWTAIL_MANDATORY if conn_data["mandatory"] else ARROWTAIL_OPTIONAL
+            arrowhead = ARROWHEAD_MANDATORY if conn_data["mandatory"] else ARROWHEAD_OPTIONAL
+            label = f'"{conn_data["label"]}<br><small><i>{conn_data["conn_type"]}</i></small>"'
+            conn_string = f"{states[from_comp]} {arrowtail} {label} {arrowhead} {states[to_comp]}"
+            connections_list.append(conn_string)
+
+    input_connections = [
+        f'i{{&ast;}}--"{conn_data["label"]}<br><small><i>{conn_data["conn_type"]}</i></small>"--> {states[to_comp]}'
+        for _, to_comp, conn_data in graph.out_edges("input", data=True)
+    ]
+    output_connections = [
+        f'{states[from_comp]}--"{conn_data["label"]}<br><small><i>{conn_data["conn_type"]}</i></small>"--> o{{&ast;}}'
+        for from_comp, _, conn_data in graph.in_edges("output", data=True)
+    ]
+    connections = "\n".join(connections_list + input_connections + output_connections)
+
+    # Create legend
+    legend_nodes = []
+    if super_component_colors:
+        legend_nodes.append("subgraph Legend")
+        for super_comp in super_component_colors:
+            legend_id = f"legend_{super_comp}"
+            legend_nodes.append(f'{legend_id}["{super_comp}"]:::{super_comp}')
+        legend_nodes.append("end")
+        connections += "\n" + "\n".join(legend_nodes)
+
+    # Add style definitions to the template
+    graph_styled = MERMAID_STYLED_TEMPLATE.format(
+        params=init_params, connections=connections, style_definitions="\n".join(style_definitions)
+    )
+    logger.debug("Mermaid diagram:\n{diagram}", diagram=graph_styled)
+
+    return graph_styled

haystack/core/pipeline/utils.py

@@ -0,0 +1,207 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import heapq
+from collections.abc import Callable
+from copy import deepcopy
+from functools import wraps
+from itertools import count
+from typing import Any
+
+from haystack import logging
+from haystack.core.component import Component
+
+logger = logging.getLogger(__name__)
+
+
+def _deepcopy_with_exceptions(obj: Any) -> Any:
+    """
+    Attempts to perform a deep copy of the given object.
+
+    This function recursively handles common container types (lists, tuples, sets, and dicts) to ensure deep copies
+    of nested structures. For specific object types that are known to be problematic for deepcopying-such as
+    instances of `Component`, `Tool`, or `Toolset` - the original object is returned as-is.
+    If `deepcopy` fails for any other reason, the original object is returned and a log message is recorded.
+
+    :param obj: The object to be deep-copied.
+
+    :returns:
+        A deep-copied version of the object, or the original object if deepcopying fails.
+    """
+    # Import here to avoid circular imports
+    from haystack.tools.tool import Tool
+    from haystack.tools.toolset import Toolset
+
+    if isinstance(obj, (list, tuple, set)):
+        return type(obj)(_deepcopy_with_exceptions(v) for v in obj)
+
+    if isinstance(obj, dict):
+        return {k: _deepcopy_with_exceptions(v) for k, v in obj.items()}
+
+    # Components and Tools often contain objects that we do not want to deepcopy or are not deepcopyable
+    # (e.g. models, clients, etc.). In this case we return the object as-is.
+    if isinstance(obj, (Component, Tool, Toolset)):
+        return obj
+
+    try:
+        return deepcopy(obj)
+    except Exception as e:
+        logger.info(
+            "Deepcopy failed for object of type '{obj_type}'. Error: {error}. Returning original object instead.",
+            obj_type=type(obj).__name__,
+            error=e,
+        )
+        return obj
+
+
+def parse_connect_string(connection: str) -> tuple[str, str | None]:
+    """
+    Returns component-connection pairs from a connect_to/from string.
+
+    :param connection:
+        The connection string.
+    :returns:
+        A tuple containing the component name and the connection name.
+    """
+    if "." in connection:
+        split_str = connection.split(".", maxsplit=1)
+        return (split_str[0], split_str[1])
+    return connection, None
+
+
+class FIFOPriorityQueue:
+    """
+    A priority queue that maintains FIFO order for items of equal priority.
+
+    Items with the same priority are processed in the order they were added.
+    This queue ensures that when multiple items share the same priority level,
+    they are dequeued in the same order they were enqueued (First-In-First-Out).
+    """
+
+    def __init__(self) -> None:
+        """
+        Initialize a new FIFO priority queue.
+        """
+        # List of tuples (priority, count, item) where count ensures FIFO order
+        self._queue: list[tuple[int, int, Any]] = []
+        # Counter to maintain insertion order for equal priorities
+        self._counter = count()
+
+    def push(self, item: Any, priority: int) -> None:
+        """
+        Push an item into the queue with a given priority.
+
+        Items with equal priority maintain FIFO ordering based on insertion time.
+        Lower priority numbers are dequeued first.
+
+        :param item:
+            The item to insert into the queue.
+        :param priority:
+            Priority level for the item. Lower numbers indicate higher priority.
+        """
+        next_count = next(self._counter)
+        entry = (priority, next_count, item)
+        heapq.heappush(self._queue, entry)
+
+    def pop(self) -> tuple[int, Any]:
+        """
+        Remove and return the highest priority item from the queue.
+
+        For items with equal priority, returns the one that was inserted first.
+
+        :returns:
+            A tuple containing (priority, item) with the lowest priority number.
+        :raises IndexError:
+            If the queue is empty.
+        """
+        if not self._queue:
+            raise IndexError("pop from empty queue")
+        priority, _, item = heapq.heappop(self._queue)
+        return priority, item
+
+    def peek(self) -> tuple[int, Any]:
+        """
+        Return but don't remove the highest priority item from the queue.
+
+        For items with equal priority, returns the one that was inserted first.
+
+        :returns:
+            A tuple containing (priority, item) with the lowest priority number.
+        :raises IndexError:
+            If the queue is empty.
+        """
+        if not self._queue:
+            raise IndexError("peek at empty queue")
+        priority, _, item = self._queue[0]
+        return priority, item
+
+    def get(self) -> tuple[int, Any] | None:
+        """
+        Remove and return the highest priority item from the queue.
+
+        For items with equal priority, returns the one that was inserted first.
+        Unlike pop(), returns None if the queue is empty instead of raising an exception.
+
+        :returns:
+            A tuple containing (priority, item), or None if the queue is empty.
+        """
+        if not self._queue:
+            return None
+        priority, _, item = heapq.heappop(self._queue)
+        return priority, item
+
+    def __len__(self) -> int:
+        """
+        Return the number of items in the queue.
+
+        :returns:
+            The number of items currently in the queue.
+        """
+        return len(self._queue)
+
+    def __bool__(self) -> bool:
+        """
+        Return True if the queue has items, False if empty.
+
+        :returns:
+            True if the queue contains items, False otherwise.
+        """
+        return bool(self._queue)
+
+
+def args_deprecated(func: Callable[..., Any]) -> Callable[..., Any]:
+    """
+    Decorator to warn about the use of positional arguments in a function.
+
+    Adapted from https://stackoverflow.com/questions/68432070/
+    :param func:
+    """
+
+    def _positional_arg_warning() -> None:
+        """
+        Triggers a warning message if positional arguments are used in a function
+        """
+        import warnings
+
+        msg = (
+            "Warning: In an upcoming release, this method will require keyword arguments for all parameters. "
+            "Please update your code to use keyword arguments to ensure future compatibility. "
+        )
+        warnings.warn(msg, DeprecationWarning, stacklevel=2)
+
+    @wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        # call the function first, to make sure the signature matches
+        ret_value = func(*args, **kwargs)
+
+        # A Pipeline instance is always the first argument - remove it from the args to check for positional arguments
+        # We check the class name as strings to avoid circular imports
+        if args and isinstance(args, tuple) and args[0].__class__.__name__ in ["Pipeline", "PipelineBase"]:
+            args = args[1:]
+
+        if args:
+            _positional_arg_warning()
+        return ret_value
+
+    return wrapper