chore: import upstream snapshot with attribution

.clusterfuzzlite/Dockerfile

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# ClusterFuzzLite / OSS-Fuzz build environment for the Haystack Atheris fuzz targets.
+# Pinned by digest for supply-chain integrity. Bump periodically (the OSS-Fuzz
+# base-builder is updated frequently with toolchain fixes); resolve a fresh digest with:
+#   docker buildx imagetools inspect gcr.io/oss-fuzz-base/base-builder-python:latest --format '{{.Manifest.Digest}}'
+FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:f45b69da82d8362d6cda092a6684cb25529c473699a55f7a8f58801f71f78cb4
+
+COPY . $SRC/haystack
+WORKDIR $SRC/haystack
+COPY .clusterfuzzlite/build.sh $SRC/

.clusterfuzzlite/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -eu
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Builds the Atheris fuzz targets for ClusterFuzzLite / OSS-Fuzz.
+# `compile_python_fuzzer` is provided by the base-builder-python image.
+
+python3 -m pip install --upgrade pip
+pip3 install . --uploaded-prior-to=P1D
+
+for harness in "$SRC"/haystack/test/fuzz/fuzz_*.py; do
+  name=$(basename "$harness" .py)
+  # --collect-submodules numpy: PyInstaller's static analysis misses NumPy's
+  # dynamically-loaded submodules (e.g. numpy._core._exceptions), which makes
+  # the frozen binary crash on startup ("ModuleNotFoundError") and fails the
+  # ClusterFuzzLite bad-build check. Bundling all numpy submodules avoids that.
+  compile_python_fuzzer "$harness" --collect-submodules numpy
+
+  # Ship a seed corpus if one exists for this harness. The runner unpacks
+  # "<fuzzer>_seed_corpus.zip" next to the binary and seeds libFuzzer with it.
+  corpus_dir="$SRC/haystack/test/fuzz/corpus/$name"
+  if [ -d "$corpus_dir" ]; then
+    zip -j "$OUT/${name}_seed_corpus.zip" "$corpus_dir"/*
+  fi
+done

.clusterfuzzlite/project.yaml

@@ -0,0 +1,5 @@
+# Project configuration for ClusterFuzzLite (and a starting point for OSS-Fuzz).
+# https://google.github.io/clusterfuzzlite/
+language: python
+sanitizers:
+  - address

.github/CODEOWNERS

@@ -0,0 +1,14 @@
+# See https://help.github.com/articles/about-codeowners/ for syntax
+
+# Core Engineering will be the default owners for everything
+# in the repo. Unless a later match takes precedence,
+# @deepset-ai/core-engineering will be requested for review
+# when someone opens a pull request.
+*                       @deepset-ai/open-source-engineering
+
+# Documentation
+*.md                    @deepset-ai/documentation @deepset-ai/open-source-engineering
+
+# Auto-synced API reference (no human reviewers needed, auto-approved by GitHub bot)
+docs-website/reference/
+docs-website/reference_versioned_docs/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Errors you encountered
+title: ''
+labels: ''
+assignees: ''
+type: Bug
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Error message**
+Error that was thrown (if available)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.
+
+**To Reproduce**
+Steps to reproduce the behavior
+
+**FAQ Check**
+- [ ] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?
+
+**System:**
+- OS:
+- Haystack version:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Something unclear? Just ask :)
+    url: https://github.com/deepset-ai/haystack/discussions/new
+    about: Start a Github discussion with your question

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,21 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+type: Feature
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,44 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'daily'
+    cooldown:
+      default-days: 1
+
+  - package-ecosystem: 'pip'
+    directory: '/'
+    schedule:
+      interval: 'daily'
+    cooldown:
+      default-days: 1
+
+  - package-ecosystem: 'npm'
+    directory: '/docs-website'
+    schedule:
+      interval: 'daily'
+    cooldown:
+      default-days: 1
+
+  # Tracks the digest-pinned `COPY --from` image (ghcr.io/astral-sh/uv) in
+  # docker/Dockerfile.base. Note: the python:3.12-slim base/builder digests are
+  # carried both as the build_image/base_image ARG defaults in Dockerfile.base
+  # and in docker/docker-bake.hcl (which overrides them at build time).
+  # Dependabot does NOT parse images that reach FROM via an ARG, nor the
+  # `docker run` image strings in workflows, so bump those by hand and keep the
+  # Dockerfile.base ARG defaults and docker-bake.hcl digests in sync.
+  - package-ecosystem: 'docker'
+    directory: '/docker'
+    schedule:
+      interval: 'daily'
+    cooldown:
+      default-days: 1
+
+  # Keeps the digest-pinned OSS-Fuzz base-builder in .clusterfuzzlite/Dockerfile fresh.
+  - package-ecosystem: 'docker'
+    directory: '/.clusterfuzzlite'
+    schedule:
+      interval: 'daily'
+    cooldown:
+      default-days: 1

.github/labeler.yml

@@ -0,0 +1,37 @@
+# Release lines
+1.x:
+ - base-branch: 'v1.x'
+
+# Proposals
+proposal:
+- changed-files:
+  - any-glob-to-any-file: proposals/text/*
+
+# Topics
+topic:tests:
+- changed-files:
+  - any-glob-to-any-file: ['test/**/*','test/*']
+
+topic:docker:
+- changed-files:
+  - any-glob-to-any-file: docker/*
+
+topic:CI:
+- changed-files:
+  - any-glob-to-any-file: ['.github/*','.github/**/*']
+
+topic:DX:
+- changed-files:
+  - any-glob-to-any-file: ["CONTRIBUTING.md", ".pre-commit-config.yaml",".gitignore"]
+
+topic:build/distribution:
+- changed-files:
+  - any-glob-to-any-file: pyproject.toml
+
+topic:security:
+- changed-files:
+  - any-glob-to-any-file: SECURITY.md
+
+topic:core:
+- changed-files:
+  - any-glob-to-any-file: haystack/core/**/*

.github/pull_request_template.md

@@ -0,0 +1,26 @@
+### Related Issues
+
+- fixes #issue-number
+
+### Proposed Changes:
+
+ <!--- In case of a bug: Describe what caused the issue and how you solved it -->
+ <!--- In case of a feature: Describe what did you add and how it works -->
+
+### How did you test it?
+
+<!-- unit tests, integration tests, manual verification, instructions for manual tests -->
+
+### Notes for the reviewer
+
+<!-- E.g. point out section where the reviewer  -->
+
+### Checklist
+
+- I have read the [contributors guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/deepset-ai/haystack/blob/main/code_of_conduct.txt).
+- I have updated the related issue with new insights and changes.
+- I have added unit tests and updated the docstrings.
+- I've used one of the [conventional commit types](https://www.conventionalcommits.org/en/v1.0.0/) for my PR title: `fix:`, `feat:`, `build:`, `chore:`, `ci:`, `docs:`, `style:`, `refactor:`, `perf:`, `test:` and added `!` in case the PR includes breaking changes.
+- I have documented my code.
+- I have added a release note file, following the [contributors guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md#release-notes).
+- I have run [pre-commit hooks](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md#installation) and fixed any issue.

.github/utils/check_imports.py

@@ -0,0 +1,85 @@
+import importlib
+import os
+import sys
+import traceback
+from pathlib import Path
+
+from haystack import logging  # noqa: F401 # this is needed to avoid circular imports
+
+
+def validate_module_imports(root_dir: str, exclude_subdirs: list[str] | None = None) -> tuple[list, list]:
+    """
+    Recursively search for all Python modules and attempt to import them.
+
+    This includes both packages (directories with __init__.py) and individual Python files.
+    """
+    imported = []
+    failed = []
+    exclude_subdirs = (exclude_subdirs or []) + ["__pycache__"]
+
+    # Add the root directory to the Python path
+    sys.path.insert(0, root_dir)
+    base_path = Path(root_dir)
+
+    for root, _, files in os.walk(root_dir):
+        if any(subdir in root for subdir in exclude_subdirs):
+            continue
+
+        # Convert path to module format
+        module_path = ".".join(Path(root).relative_to(base_path.parent).parts)
+        python_files = [f for f in files if f.endswith(".py")]
+
+        # Try importing package and individual files
+        for file in python_files:
+            try:
+                if file == "__init__.py":
+                    module_to_import = module_path
+                else:
+                    module_name = os.path.splitext(file)[0]
+                    module_to_import = f"{module_path}.{module_name}" if module_path else module_name
+
+                importlib.import_module(module_to_import)
+                imported.append(module_to_import)
+            except Exception:
+                failed.append({"module": module_to_import, "traceback": traceback.format_exc()})
+
+    return imported, failed
+
+
+def main() -> None:
+    """
+    This script checks that all Haystack modules can be imported successfully.
+
+    This includes both packages and individual Python files.
+    This can detect several issues, such as:
+    - Syntax errors in Python files
+    - Missing dependencies
+    - Circular imports
+    - Incorrect type hints without forward references
+    """
+    # Add any subdirectories you want to skip during import checks ("__pycache__" is skipped by default)
+    exclude_subdirs = ["testing"]
+
+    print("Checking imports from all Haystack modules...")
+    imported, failed = validate_module_imports(root_dir="haystack", exclude_subdirs=exclude_subdirs)
+
+    if not imported:
+        print("\nNO MODULES WERE IMPORTED")
+        sys.exit(1)
+
+    print(f"\nSUCCESSFULLY IMPORTED {len(imported)} MODULES")
+
+    if failed:
+        print(f"\nFAILED TO IMPORT {len(failed)} MODULES:")
+        for fail in failed:
+            print(f"  - {fail['module']}")
+
+        print("\nERRORS:")
+        for fail in failed:
+            print(f"  - {fail['module']}\n")
+            print(f"    {fail['traceback']}\n\n")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

.github/utils/create_unstable_docs_docusaurus.py

@@ -0,0 +1,102 @@
+"""
+This script creates an unstable documentation version at the time of branch-off for a new Haystack release.
+
+Between branch-off and the actual release, two unstable doc versions coexist.
+If we branch off for 2.20, we have:
+1. the target unstable version, 2.20-unstable (lives in docs-website/versioned_docs/version-2.20-unstable)
+2. the next unstable version, 2.21-unstable (lives in docs-website/docs)
+
+This script takes care of all the necessary updates to the documentation website.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--new-version", help="The new unstable version that is being created (e.g. 2.20).", required=True
+    )
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.new_version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.new_version}"  # e.g., "2.20" - the target release version
+    major, minor = args.new_version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    next_unstable = f"{major}.{int(minor) + 1}-unstable"  # e.g., "2.21-unstable" - next cycle
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    # Check if the versions we're about to create already exist in versioned_docs
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if target_unstable in versions:
+        print(f"{target_unstable} already exists. Nothing to do.")
+        sys.exit(0)
+
+    # Create new unstable from the currently existing one.
+    # The new unstable will be made stable at a later time by another workflow
+    print(f"Creating new unstable version {target_unstable} from main")
+
+    ### Docusaurus updates
+
+    # copy docs to versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/docs", f"docs-website/versioned_docs/version-{target_unstable}")
+
+    # copy reference to reference_versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/reference", f"docs-website/reference_versioned_docs/version-{target_unstable}")
+
+    # generate versioned_sidebars/version-target_unstable-sidebars.json from the current sidebars.js
+    with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+        tmp_path = tmp.name
+    subprocess.run(
+        ["node", "docs-website/scripts/extract_sidebar.mjs", "docs-website/sidebars.js", tmp_path], check=True
+    )
+    docs_sidebar_dest = f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json"
+    shutil.move(tmp_path, docs_sidebar_dest)
+
+    # generate reference_versioned_sidebars/version-target_unstable-sidebars.json from the current reference-sidebars.js
+    ref_sidebar_dest = f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json"
+    with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+        tmp_path = tmp.name
+    subprocess.run(
+        ["node", "docs-website/scripts/extract_sidebar.mjs", "docs-website/reference-sidebars.js", tmp_path], check=True
+    )
+    shutil.move(tmp_path, ref_sidebar_dest)
+
+    # add unstable version to versions.json
+    with open("docs-website/versions.json") as f:
+        versions_list = json.load(f)
+    versions_list.insert(0, target_unstable)
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # add unstable version to reference_versions.json
+    with open("docs-website/reference_versions.json") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list.insert(0, target_unstable)
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace the target unstable version with the next unstable version
+    with open("docs-website/docusaurus.config.js") as f:
+        config = f.read()
+    config = config.replace(f"label: '{target_unstable}'", f"label: '{next_unstable}'")
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)

.github/utils/docs_search_sync.py

@@ -0,0 +1,120 @@
+"""
+This script syncs the Haystack docs HTML files to the deepset workspace for search indexing.
+
+It is used in the docs_search_sync.yml workflow.
+
+1. Collects all HTML files from the docs and reference directories for the stable Haystack version.
+2. Uploads the HTML files to the deepset workspace.
+    - A timestamp-based metadata field is used to track document versions in the workspace.
+3. Deletes the old HTML files from the deepset workspace.
+    - Since most files are overwritten during upload, only a small number of deletions is expected.
+    - In case MAX_DELETIONS_SAFETY_LIMIT is exceeded, we block the deletion.
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+from deepset_cloud_sdk.workflows.sync_client.files import DeepsetCloudFile, WriteMode, list_files, upload_texts
+
+DEEPSET_WORKSPACE_DOCS_SEARCH = os.environ["DEEPSET_WORKSPACE_DOCS_SEARCH"]
+DEEPSET_API_KEY_DOCS_SEARCH = os.environ["DEEPSET_API_KEY_DOCS_SEARCH"]
+
+# If there are more files to delete than this limit, it's likely that something went wrong in the upload process.
+MAX_DELETIONS_SAFETY_LIMIT = 20
+
+
+def collect_docs_files(version: int) -> list[DeepsetCloudFile]:
+    """
+    Collect all HTML files from the docs and reference directories.
+
+    Returns a list of DeepsetCloudFile objects.
+    """
+    repo_root = Path(__file__).parent.parent.parent
+    build_dir = repo_root / "docs-website" / "build"
+    # we want to exclude previous and temporarily unstable versions (2.x) and next version (next)
+    exclude = ("2.", "next")
+
+    files = []
+    for section in ("docs", "reference"):
+        for subfolder in (build_dir / section).iterdir():
+            if subfolder.is_dir() and not any(x in subfolder.name for x in exclude):
+                for html_file in subfolder.rglob("*.html"):
+                    files.append(
+                        DeepsetCloudFile(
+                            # The build produces files like docs/agents/index.html or reference/agents-api/index.html.
+                            # For file names, we want to use the parent directory name (agents.html or agents-api.html)
+                            name=f"{html_file.parent.name}.html",
+                            text=html_file.read_text(),
+                            meta={
+                                "type": "api-reference" if section == "reference" else "documentation",
+                                "version": version,
+                            },
+                        )
+                    )
+    return files
+
+
+def delete_files(file_names: list[str]) -> None:
+    """
+    Delete files from the deepset workspace.
+    """
+    url = f"https://api.cloud.deepset.ai/api/v1/workspaces/{DEEPSET_WORKSPACE_DOCS_SEARCH}/files"
+    payload = {"names": file_names}
+    headers = {"Accept": "application/json", "Authorization": f"Bearer {DEEPSET_API_KEY_DOCS_SEARCH}"}
+    response = requests.delete(url, json=payload, headers=headers, timeout=300)
+    response.raise_for_status()
+
+
+if __name__ == "__main__":
+    version = time.time_ns()
+    print(f"Docs version: {version}")
+
+    print("Collecting docs files from build directory")
+    dc_files = collect_docs_files(version)
+    print(f"Collected {len(dc_files)} docs files")
+
+    if len(dc_files) == 0:
+        print("No docs files found. Something is wrong. Exiting.")
+        sys.exit(1)
+
+    print("Uploading docs files to deepset")
+    summary = upload_texts(
+        workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH,
+        files=dc_files,
+        api_key=DEEPSET_API_KEY_DOCS_SEARCH,
+        blocking=True,  # Very important to ensure that DC is up to date when we query for deletion
+        timeout_s=300,
+        show_progress=True,
+        write_mode=WriteMode.OVERWRITE,
+        enable_parallel_processing=True,
+    )
+    print(f"Uploaded docs files to deepset\n{summary}")
+    if summary.failed_upload_count > 0:
+        print("Failed to upload some docs files. Stopping to prevent risky deletion of old files.")
+        sys.exit(1)
+
+    print("Listing old docs files from deepset")
+    odata_filter = f"version lt '{version}'"
+    old_files_names = [
+        f.name
+        for batch in list_files(
+            workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH, api_key=DEEPSET_API_KEY_DOCS_SEARCH, odata_filter=odata_filter
+        )
+        for f in batch
+    ]
+
+    print(f"Found {len(old_files_names)} old files to delete")
+    if len(old_files_names) > MAX_DELETIONS_SAFETY_LIMIT:
+        print(
+            f"Found >{MAX_DELETIONS_SAFETY_LIMIT} old files to delete. "
+            "Stopping because something could have gone wrong in the upload process."
+        )
+        sys.exit(1)
+
+    if len(old_files_names) > 0:
+        print("Deleting old docs files from deepset")
+        delete_files(old_files_names)
+        print("Deleted old docs files from deepset")

.github/utils/docstrings_checksum.py

@@ -0,0 +1,46 @@
+import ast
+import hashlib
+from collections.abc import Iterator
+from pathlib import Path
+
+
+def docstrings_checksum(python_files: Iterator[Path]) -> str:
+    """
+    Calculate the checksum of the docstrings in the given Python files.
+    """
+    files_content = (f.read_text() for f in python_files)
+    trees = (ast.parse(c) for c in files_content)
+
+    # Get all docstrings from async functions, functions,
+    # classes and modules definitions
+    docstrings = []
+    for tree in trees:
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module)):
+                # Skip all node types that can't have docstrings to prevent failures
+                continue
+            docstring = ast.get_docstring(node)
+            if docstring:
+                docstrings.append(docstring)
+
+    # Sort them to be safe, since ast.walk() returns
+    # nodes in no specified order.
+    # See https://docs.python.org/3/library/ast.html#ast.walk
+    docstrings.sort()
+
+    return hashlib.md5(str(docstrings).encode("utf-8")).hexdigest()
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", help="Haystack root folder", required=True, type=Path)
+    args = parser.parse_args()
+
+    # Get all Haystack and rest_api python files
+    root: Path = args.root.absolute()
+    haystack_files = root.glob("haystack/**/*.py")
+
+    md5 = docstrings_checksum(haystack_files)
+    print(md5)

.github/utils/parse_validate_version.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+# parse_validate_version.sh - Parse and validate version for release
+#
+# Usage: ./parse_validate_version.sh <version>
+# Output: Writes to $GITHUB_OUTPUT if set, otherwise to stdout
+#
+# Example:
+#   ./parse_validate_version.sh v2.99.0-rc1
+#
+# This script is used in the release.yml workflow to parse and validate the version to be released.
+# Covers several checks to prevent accidental releases of incorrect versions.
+
+set -euo pipefail
+
+# --- Helpers ---
+
+fail() {
+    echo ""
+    echo -e "❌ $1"
+    echo ""
+    exit 1
+}
+
+ok() {
+    echo "✅ $1"
+}
+
+tag_exists() {
+    git tag -l "$1" | grep -q "^$1$"
+}
+
+branch_exists() {
+    git ls-remote --heads origin "$1" | grep -q "$1"
+}
+
+# --- Parse and validate version ---
+
+VERSION="${1#v}"  # Strip 'v' prefix
+
+echo ""
+echo "ℹ️  Validating: ${1}"
+echo ""
+
+if [[ ! "${VERSION}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-rc([0-9]+))?$ ]]; then
+    fail "Invalid version format: $1\n\n"\
+"Expected format: vMAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH-rcN\n"\
+"Examples: v2.99.0-rc1, v2.99.0, v2.99.1-rc1"
+fi
+ok "Version format is valid"
+
+MAJOR="${BASH_REMATCH[1]}"
+MINOR="${BASH_REMATCH[2]}"
+PATCH="${BASH_REMATCH[3]}"
+RC_NUM="${BASH_REMATCH[5]:-0}"
+
+if [[ "${RC_NUM}" == "0" && "${VERSION}" == *"-rc0" ]]; then
+    fail "Cannot release rc0\n\n"\
+"rc0 is an internal marker created automatically during branch-off.\n"\
+"Release candidates start at rc1."
+fi
+
+MAJOR_MINOR="${MAJOR}.${MINOR}"
+RELEASE_BRANCH="v${MAJOR_MINOR}.x"
+TAG="v${VERSION}"
+
+IS_RC="false"
+[[ "${RC_NUM}" != "0" ]] && IS_RC="true"
+
+IS_FIRST_RC="false"
+if [[ "${PATCH}" == "0" && "${RC_NUM}" == "1" ]]; then
+    IS_FIRST_RC="true"
+fi
+
+# 1. Tag must not already exist
+if tag_exists "${TAG}"; then
+    fail "Version ${TAG} was already released\n\n"\
+"Each version can only be released once.\n"\
+"To publish changes, release the next RC or patch version."
+fi
+ok "Tag ${TAG} does not exist"
+
+# 2. Checks based on release type
+if [[ "${IS_FIRST_RC}" == "true" ]]; then
+    # First RC of minor: branch must NOT exist yet
+    if branch_exists "${RELEASE_BRANCH}"; then
+        fail "Branch ${RELEASE_BRANCH} already exists\n\n"\
+"The first RC of a minor (e.g., v${MAJOR_MINOR}.0-rc1) creates the release branch.\n"\
+"Since the branch exists, this minor was likely already started.\n"\
+"Did you mean to release the next RC (rc2, rc3...) or a patch (v${MAJOR_MINOR}.1-rc1)?"
+    fi
+    ok "Branch ${RELEASE_BRANCH} does not exist"
+
+    # First RC of minor: VERSION.txt must contain rc0
+    EXPECTED="${MAJOR_MINOR}.0-rc0"
+    ACTUAL=$(cat VERSION.txt)
+    if [[ "${ACTUAL}" != "${EXPECTED}" ]]; then
+        ACTUAL_MINOR=$(echo "${ACTUAL}" | cut -d. -f1,2)
+        fail "Cannot release v${MAJOR_MINOR}.0-rc1 from this branch\n\n"\
+"The main branch is prepared for version ${ACTUAL_MINOR}, not ${MAJOR_MINOR}.\n"\
+"Check that you're releasing the correct version."
+    fi
+    ok "VERSION.txt = ${EXPECTED}"
+
+else
+    # Not first RC: branch MUST exist
+    if ! branch_exists "${RELEASE_BRANCH}"; then
+        if [[ "${PATCH}" == "0" ]]; then
+            fail "Branch ${RELEASE_BRANCH} does not exist\n\n"\
+"For subsequent RCs (rc2, rc3...), the release branch must already exist.\n"\
+"Release the first RC (v${MAJOR_MINOR}.0-rc1) first to create the branch."
+        else
+            fail "Branch ${RELEASE_BRANCH} does not exist\n\n"\
+"For patch releases, the release branch must already exist.\n"\
+"The minor version (v${MAJOR_MINOR}.0) must be released before any patches."
+        fi
+    fi
+    ok "Branch ${RELEASE_BRANCH} exists"
+
+    # Subsequent RC (rc2, rc3...): previous RC must exist
+    if [[ "${RC_NUM}" -gt 1 ]]; then
+        PREV_RC_NUM=$((RC_NUM - 1))
+        PREV_TAG="v${MAJOR_MINOR}.${PATCH}-rc${PREV_RC_NUM}"
+        if ! tag_exists "${PREV_TAG}"; then
+            fail "Cannot release v${MAJOR_MINOR}.${PATCH}-rc${RC_NUM}\n\n"\
+"Previous RC (${PREV_TAG}) was not found.\n"\
+"RC versions must be sequential. Release rc${PREV_RC_NUM} first."
+        fi
+        ok "Previous tag ${PREV_TAG} exists"
+    fi
+
+    # Final release: at least one RC must exist
+    if [[ "${RC_NUM}" == "0" ]]; then
+        RC_TAGS=$(git tag -l "v${MAJOR_MINOR}.${PATCH}-rc*" | grep -v "\-rc0$" || true)
+        if [[ -z "${RC_TAGS}" ]]; then
+            fail "Cannot release stable version v${MAJOR_MINOR}.${PATCH}\n\n"\
+"No release candidate found for this version.\n"\
+"Stable releases require at least one RC first (e.g., v${MAJOR_MINOR}.${PATCH}-rc1)."
+        fi
+        LAST_RC=$(echo "${RC_TAGS}" | sort -V | tail -n1)
+        ok "Found RC: ${LAST_RC}"
+
+        # Check Tests workflow passed (only if credentials available)
+        if [[ -n "${GH_TOKEN:-}" && -n "${GITHUB_REPOSITORY:-}" ]]; then
+            RC_SHA=$(git rev-list -n 1 "${LAST_RC}")
+            RESULT=$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${RC_SHA}&status=success" \
+                --jq '.workflow_runs[] | select(.name == "Tests") | .conclusion' 2>/dev/null || true)
+            if [[ -z "${RESULT}" ]]; then
+                fail "Cannot release stable version v${MAJOR_MINOR}.${PATCH}\n\n"\
+"Tests did not pass on the last RC (${LAST_RC}).\n"\
+"Wait for tests to complete, or release a new RC with fixes."
+            fi
+            ok "Tests passed on ${LAST_RC}"
+        fi
+    fi
+fi
+
+echo ""
+ok "All validations passed!"
+echo ""
+
+# --- Output to GITHUB_OUTPUT (or stdout for local testing) ---
+
+OUTPUT_FILE="${GITHUB_OUTPUT:-/dev/stdout}"
+
+{
+    echo "version=${VERSION}"
+    echo "major_minor=${MAJOR_MINOR}"
+    echo "release_branch=${RELEASE_BRANCH}"
+    echo "is_rc=${IS_RC}"
+    echo "is_first_rc=${IS_FIRST_RC}"
+} >> "${OUTPUT_FILE}"

.github/utils/prepare_release_notification.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# prepare_release_notification.sh - Prepare Slack notification for release outcome
+#
+# Requires: VERSION, RUN_URL, HAS_FAILURE, GH_TOKEN, GITHUB_REPOSITORY
+# Optional: IS_RC, IS_FIRST_RC, MAJOR_MINOR, GITHUB_URL, PYPI_URL, DOCKER_URL,
+#           BUMP_VERSION_PR_URL, DC_PIPELINE_TEMPLATES_PR_URL, DC_CUSTOM_NODES_PR_URL,
+#           HAYSTACK_RUNTIME_PR_URL, GITHUB_WORKSPACE
+# Output: slack_payload.json
+#
+# This script is used in the release.yml workflow to prepare the notification payload
+# sent to Slack after a release completes (success or failure).
+# Text uses Slack mrkdwn format: *bold*, <url|label> for links.
+
+set -euo pipefail
+
+PAYLOAD_FILE="${GITHUB_WORKSPACE:-/tmp}/slack_payload.json"
+
+write_payload() {
+  jq -n --arg text "$TXT" '{
+    text: $text,
+    blocks: [{ type: "section", text: { type: "mrkdwn", text: $text } }]
+  }' > "$PAYLOAD_FILE"
+}
+
+if [[ "${HAS_FAILURE}" == "true" ]]; then
+  TXT=":red_circle: Release *${VERSION}* failed"
+  TXT+=$'\n'"Check workflow run for details: <${RUN_URL}|View Logs>"
+  write_payload
+  exit 0
+fi
+
+# Success case
+
+TXT=":white_check_mark: Release *${VERSION}* completed successfully"
+
+# Add artifact URLs if available
+if [[ -n "${GITHUB_URL:-}" || -n "${PYPI_URL:-}" || -n "${DOCKER_URL:-}" ]]; then
+  TXT+=$'\n\n:package: *Artifacts:*'
+  [[ -n "${GITHUB_URL:-}" ]] && TXT+=$'\n'"- <${GITHUB_URL}|Release notes (GitHub)>"
+  [[ -n "${PYPI_URL:-}" ]] && TXT+=$'\n'"- <${PYPI_URL}|PyPI>"
+  [[ -n "${DOCKER_URL:-}" ]] && TXT+=$'\n'"- <${DOCKER_URL}|Docker>"
+fi
+
+# For RCs, include link to the Tests workflow run
+if [[ "${IS_RC:-}" == "true" ]]; then
+  COMMIT_SHA=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${VERSION}" --jq '.sha' 2>/dev/null || echo "")
+  if [[ -n "${COMMIT_SHA}" ]]; then
+    TESTS_RUN=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${COMMIT_SHA}" \
+      --jq '.workflow_runs[] | select(.name == "Tests") | .html_url' 2>/dev/null | head -1 || echo "")
+    if [[ -n "${TESTS_RUN}" ]]; then
+      TXT+=$'\n\n'":test_tube: <${TESTS_RUN}|Haystack Tests>"
+    fi
+  fi
+fi
+
+# For first RC, include the PRs to merge from branch-off
+if [[ "${IS_FIRST_RC:-}" == "true" && -n "${BUMP_VERSION_PR_URL:-}" ]]; then
+  TXT+=$'\n\n'":clipboard: *PRs to merge:*"
+  TXT+=$'\n'"- <${BUMP_VERSION_PR_URL}|Bump unstable version and create unstable docs>"
+fi
+
+# For RCs, include Platform test PRs
+if [[ "${IS_RC:-}" == "true" ]]; then
+  PLATFORM_PRS=""
+  [[ -n "${DC_PIPELINE_TEMPLATES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_PIPELINE_TEMPLATES_PR_URL}|dc-pipeline-templates>"
+  [[ -n "${DC_CUSTOM_NODES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_CUSTOM_NODES_PR_URL}|deepset-cloud-custom-nodes>"
+  [[ -n "${HAYSTACK_RUNTIME_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${HAYSTACK_RUNTIME_PR_URL}|haystack-runtime>"
+  if [[ -n "${PLATFORM_PRS}" ]]; then
+    TXT+=$'\n\n'":factory: *Test PRs opened on Platform:*${PLATFORM_PRS}"
+  fi
+fi
+
+# For RCs, request Platform Engineering taking over testing
+if [[ "${IS_RC:-}" == "true" ]]; then
+  TXT+=$'\n\n'"This release is marked as a Release Candidate."
+  TXT+=$'\n'"Notify #deepset-platform-engineering channel on Slack that the RC is available"
+  TXT+=" and that Platform tests pass/fail, linking the PRs opened on the Platform repositories."
+fi
+
+# For final minor releases (vX.Y.0), include the docs promotion PR
+if [[ "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.0$ && -n "${MAJOR_MINOR:-}" ]]; then
+  PROMOTE_DOCS_PR_URL=$(gh pr list --repo "${GITHUB_REPOSITORY}" \
+    --head "promote-unstable-docs-${MAJOR_MINOR}" --json url --jq '.[0].url' 2>/dev/null || echo "")
+  if [[ -n "${PROMOTE_DOCS_PR_URL}" ]]; then
+    TXT+=$'\n\n'":clipboard: *PRs to merge:*"
+    TXT+=$'\n'"- <${PROMOTE_DOCS_PR_URL}|Promote unstable docs>"
+  fi
+fi
+
+# For final releases (not RCs), include info about pushing release notes to website
+if [[ "${IS_RC:-}" != "true" ]]; then
+  TXT+=$'\n\n'":memo: After refining and finalizing release notes, push them to Haystack website:"
+  TXT+=$'\n'"\`gh workflow run push_release_notes_to_website.yml -R deepset-ai/haystack -f version=${VERSION}\`"
+fi
+
+write_payload

.github/utils/promote_unstable_docs_docusaurus.py

@@ -0,0 +1,117 @@
+"""
+This script promotes an unstable documentation version to a stable version at the time of a new Haystack release.
+
+To understand how unstable doc versions are created, see create_unstable_docs_docusaurus.py.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+MAX_STABLE_VERSIONS = 5
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", help="The version to promote to stable (e.g. 2.20).", required=True)
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.version}"  # e.g., "2.20" - the target release version
+    major, minor = args.version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    previous_stable = f"{major}.{int(minor) - 1}"  # e.g., "2.19" - previous stable release
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if target_unstable not in versions:
+        sys.exit(f"Can't find version {target_unstable} to promote to {target_version}")
+
+    print(f"Promoting unstable version {target_unstable} to stable version {target_version}")
+
+    ### Docusaurus updates
+
+    # move versioned_docs/version-target_unstable to versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/versioned_docs/version-{target_unstable}",
+        f"docs-website/versioned_docs/version-{target_version}",
+    )
+
+    # move reference_versioned_docs/version-target_unstable to reference_versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/reference_versioned_docs/version-{target_unstable}",
+        f"docs-website/reference_versioned_docs/version-{target_version}",
+    )
+
+    # move versioned_sidebars/version-target_unstable-sidebars.json
+    # to versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # move reference_versioned_sidebars/version-target_unstable-sidebars.json
+    # to reference_versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/reference_versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # replace unstable version with stable version in versions.json
+    with open("docs-website/versions.json") as f:
+        versions_list = json.load(f)
+    versions_list[versions_list.index(target_unstable)] = target_version
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # replace unstable version with stable version in reference_versions.json
+    with open("docs-website/reference_versions.json") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list[reference_versions_list.index(target_unstable)] = target_version
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace previous stable version with the target version
+    with open("docs-website/docusaurus.config.js") as f:
+        config = f.read()
+    config = config.replace(f"lastVersion: '{previous_stable}'", f"lastVersion: '{target_version}'")  # "2.19" -> "2.20"
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)
+
+    # regenerate vercel.json redirects for inactive versions (those beyond the top MAX_STABLE_VERSIONS)
+    with open("docs-website/versions.json") as f:
+        updated_versions = json.load(f)
+    stable_versions = [v for v in updated_versions if not v.endswith("-unstable")]
+    inactive_versions = stable_versions[MAX_STABLE_VERSIONS:]
+    redirects = []
+    for v in inactive_versions:
+        redirects.append({"source": f"/docs/{v}/:slug*", "destination": "/docs/:slug*", "permanent": True})
+        redirects.append({"source": f"/reference/{v}/:slug*", "destination": "/reference/:slug*", "permanent": True})
+
+    with open("docs-website/vercel.json") as f:
+        vercel_config = json.load(f)
+
+    existing_redirects = vercel_config.get("redirects", [])
+    existing_sources = {r.get("source") for r in existing_redirects}
+
+    for r in redirects:
+        if r["source"] not in existing_sources:
+            existing_redirects.append(r)
+
+    vercel_config["redirects"] = existing_redirects
+    with open("docs-website/vercel.json", "w") as f:
+        json.dump(vercel_config, f, indent=2)
+        f.write("\n")
+    print(f"Updated vercel.json with {len(redirects)} redirect(s) for inactive versions: {inactive_versions}")

.github/utils/pyproject_to_requirements.py

@@ -0,0 +1,53 @@
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import toml
+
+matcher = re.compile(r"farm-haystack\[(.+)\]")
+parser = argparse.ArgumentParser(
+    prog="pyproject_to_requirements.py", description="Convert pyproject.toml to requirements.txt"
+)
+parser.add_argument("pyproject_path")
+parser.add_argument("--extra", default="")
+
+
+def resolve(target: str, extras: dict, results: set) -> None:
+    """
+    Resolve the dependencies for a given target.
+    """
+    if target not in extras:
+        results.add(target)
+        return
+
+    for t in extras[target]:
+        m = matcher.match(t)
+        if m:
+            for i in m.group(1).split(","):
+                resolve(i, extras, results)
+        else:
+            resolve(t, extras, results)
+
+
+def main(pyproject_path: Path, extra: str = "") -> None:
+    """
+    Convert a pyproject.toml file to a requirements.txt file.
+    """
+    content = toml.load(pyproject_path)
+    # basic set of dependencies
+    deps = set(content["project"]["dependencies"])
+
+    if extra:
+        extras = content["project"]["optional-dependencies"]
+        resolve(extra, extras, deps)
+
+    sys.stdout.write("\n".join(sorted(deps)))
+    sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    pyproject_path = Path(args.pyproject_path).absolute()
+
+    main(pyproject_path, args.extra)

.github/utils/update_haystack_dc_custom_nodes.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Update the haystack-ai version in the deepset-cloud-custom-nodes uv.lock file.
+
+Fetches sdist/wheel hashes from PyPI and updates the haystack-ai package entry,
+preserving the existing lock file formatting.
+"""
+
+import argparse
+import json
+import sys
+import urllib.request
+
+import tomlkit
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version", help="Version to update to (e.g. 2.26.1-rc1)")
+    parser.add_argument("lock_file", help="Path to uv.lock")
+    args = parser.parse_args()
+
+    # PEP 440 normalized version for filenames
+    new_version = args.version.replace("-", "")
+
+    # Fetch hashes from PyPI
+    pypi_data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/haystack-ai/{args.version}/json"))
+    sdist_sha = wheel_sha = None
+    for u in pypi_data["urls"]:
+        if u["packagetype"] == "sdist":
+            sdist_sha = u["digests"]["sha256"]
+        elif u["packagetype"] == "bdist_wheel":
+            wheel_sha = u["digests"]["sha256"]
+    if not sdist_sha or not wheel_sha:
+        sys.exit("Could not find sdist or wheel hashes on PyPI")
+
+    with open(args.lock_file) as f:
+        data = tomlkit.load(f)
+
+    found = False
+    for pkg in data["package"]:
+        if pkg["name"] == "haystack-ai":
+            old_version = pkg["version"]
+
+            pkg["version"] = new_version
+
+            pkg["sdist"]["url"] = pkg["sdist"]["url"].replace(old_version, new_version)
+            pkg["sdist"]["hash"] = f"sha256:{sdist_sha}"
+
+            wheel = pkg["wheels"][0]
+            wheel["url"] = wheel["url"].replace(old_version, new_version)
+            wheel["hash"] = f"sha256:{wheel_sha}"
+
+            found = True
+            print(f"Updated haystack-ai from {old_version} to {new_version}")
+            break
+
+    if not found:
+        sys.exit("haystack-ai package not found in uv.lock")
+
+    with open(args.lock_file, "w") as f:
+        tomlkit.dump(data, f)

.github/utils/wait_for_workflows.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# wait_for_workflows.sh - Wait for tag-triggered workflows to complete
+#
+# Usage: ./wait_for_workflows.sh <tag> <workflow_name1> [workflow_name2] ...
+# Requires: GH_TOKEN and GITHUB_REPOSITORY environment variables
+#
+# Example:
+#   ./wait_for_workflows.sh v2.19.0 "Project release on PyPi" "Docker image release"
+#
+# This script is used in the release.yml workflow to wait for the workflows triggered by a specific release tag to
+# successfully complete.
+
+
+# With the default values, we wait for 20 minutes
+MAX_ATTEMPTS="${MAX_ATTEMPTS:-40}"
+SLEEP_SECONDS="${SLEEP_SECONDS:-30}"
+
+set -euo pipefail
+
+if [[ -z "${GH_TOKEN:-}" ]] || [[ -z "${GITHUB_REPOSITORY:-}" ]]; then
+    echo "❌ GH_TOKEN and GITHUB_REPOSITORY must be set"
+    exit 1
+fi
+
+TAG="$1"
+shift
+WORKFLOWS=("$@")
+
+
+# Get commit SHA from tag
+TAG_SHA=$(git rev-list -n 1 "${TAG}" 2>/dev/null) || {
+    echo "❌ Tag ${TAG} not found"
+    exit 1
+}
+
+echo "Tag ${TAG} (commit: ${TAG_SHA:0:7})"
+echo ""
+
+wait_for_workflow() {
+    local name="$1"
+    echo "⏳ Waiting for: $name"
+
+    for ((i=1; i<=MAX_ATTEMPTS; i++)); do
+        jq_filter="[.workflow_runs[] | select(.head_sha == \"${TAG_SHA}\" and .name == \"${name}\")]
+            | sort_by(.created_at) | last"
+
+        result=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs" \
+            --jq "$jq_filter" 2>/dev/null || echo "")
+
+        if [[ -z "$result" ]]; then
+            echo "   Attempt $i/$MAX_ATTEMPTS: not started yet..."
+            sleep $SLEEP_SECONDS
+            continue
+        fi
+
+        status=$(echo "$result" | jq -r '.status')
+        conclusion=$(echo "$result" | jq -r '.conclusion')
+
+        if [[ "$status" == "completed" ]]; then
+            if [[ "$conclusion" == "success" ]]; then
+                echo "✅ $name completed"
+                return 0
+            else
+                echo "❌ $name failed: $conclusion"
+                return 1
+            fi
+        fi
+
+        echo "   Attempt $i/$MAX_ATTEMPTS: $status..."
+        sleep $SLEEP_SECONDS
+    done
+
+    echo "❌ $name: timeout after $((MAX_ATTEMPTS * SLEEP_SECONDS / 60)) minutes"
+    return 1
+}
+
+for workflow in "${WORKFLOWS[@]}"; do
+    wait_for_workflow "$workflow" || exit 1
+done
+
+echo ""
+echo "✅ All workflows completed"

.github/workflows/auto_approve_api_ref_sync.yml

@@ -0,0 +1,35 @@
+name: Approve and merge API reference sync PRs
+
+# Automatically approve and merge API reference sync PRs from Haystack, Haystack Core Integrations,
+# and Haystack Experimental
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "docs-website/reference/**"
+      - "docs-website/reference_versioned_docs/**"
+
+permissions:
+  contents: read
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  auto-approve-and-merge:
+    permissions:
+      contents: write
+      pull-requests: write
+    if: |
+      github.event.pull_request.user.login == 'HaystackBot' &&
+      startsWith(github.event.pull_request.head.ref, 'sync-docusaurus-api-reference') &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-slim
+    steps:
+      - name: Approve PR
+        run: gh pr review --approve ${{ github.event.pull_request.number }} --repo ${{ github.repository }}
+
+      - name: Enable auto-merge
+        run: gh pr merge --squash --auto ${{ github.event.pull_request.number }} --repo ${{ github.repository }}

.github/workflows/branch_off.yml

@@ -0,0 +1,99 @@
+name: Branch off
+
+on:
+  workflow_call:
+    outputs:
+      bump_version_pr_url:
+        description: 'URL of the bump version PR'
+        value: ${{ jobs.branch-off.outputs.bump_version_pr_url }}
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  branch-off:
+    runs-on: ubuntu-slim
+    outputs:
+      bump_version_pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: main
+          # Persist the bot token so the branch/tag pushes below authenticate as
+          # HaystackBot (a ruleset-bypass actor) instead of the default
+          # github-actions[bot], which only has contents:read and is blocked by
+          # the release rulesets.
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Define all versions
+        id: versions
+        shell: bash
+        run: |
+          # example: 2.20.0-rc0 in VERSION.txt -> 2.20
+          echo "current_version_major_minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"
+          # example: 2.20.0-rc0 in VERSION.txt -> 2.21.0-rc0
+          echo "next_version_rc0=$(awk -F. '/[0-9]+\./{$2++;print}' OFS=. < VERSION.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Create release branch and tag
+        shell: bash
+        env:
+          # We use the HAYSTACK_BOT_TOKEN here so the PR created by the step will
+          # trigger required workflows and can be merged by anyone
+          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create the release branch from main
+          git checkout -b v${{ steps.versions.outputs.current_version_major_minor }}.x
+          git push -u origin v${{ steps.versions.outputs.current_version_major_minor }}.x
+
+          # Tag the branch-off point with the next version rc0 to mark start of next dev cycle.
+          # The tag points to this commit (before VERSION.txt is bumped).
+          # This is intentional for reno to work properly.
+          git tag "v${{ steps.versions.outputs.next_version_rc0 }}" -m "v${{ steps.versions.outputs.next_version_rc0 }}"
+          git push --tags
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Prepare changes for main
+        shell: bash
+        run: |
+          git checkout main
+
+          # Bump VERSION.txt to next version rc0
+          echo "${{ steps.versions.outputs.next_version_rc0 }}" > VERSION.txt
+
+          # Generate unstable docs for Docusaurus
+          python ./.github/utils/create_unstable_docs_docusaurus.py --new-version ${{ steps.versions.outputs.current_version_major_minor }}
+
+      - name: Create PR to bump unstable version and create unstable docs
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Bump unstable version and create unstable docs"
+          branch: bump-version
+          base: main
+          title: "Bump unstable version and create unstable docs"
+          add-paths: |
+            VERSION.txt
+            docs-website
+          body: |
+            This PR:
+            - Bumps the unstable version to `${{ steps.versions.outputs.next_version_rc0 }}`
+            - Creates the unstable docs for Haystack ${{ steps.versions.outputs.current_version_major_minor }}
+
+            You can inspect the docs preview (two unstable versions will be available) and merge it.
+          labels: "ignore-for-release-notes"
+          reviewers: "${{ github.actor }}"

.github/workflows/cflite_pr.yml

@@ -0,0 +1,57 @@
+name: ClusterFuzzLite PR fuzzing
+
+# Short, code-change-scoped fuzzing run on PRs that touch fuzzed code or the
+# fuzzing setup. Catches regressions and crashes introduced by a change.
+# Continuous/batch fuzzing can be added later as a separate scheduled workflow.
+#
+# Scoped to the modules the harnesses actually exercise rather than all of
+# `haystack/**`, so the (build-heavy) job doesn't run on every library PR:
+#   - Pipeline.loads          -> core/pipeline, core/serialization, marshal
+#   - Document.from_dict       -> dataclasses
+#   - document_matches_filter  -> utils/filters
+# Deep, transitively-reached regressions are better caught by a scheduled batch
+# run; random YAML/JSON essentially never reaches haystack/components.
+on:
+  pull_request:
+    paths:
+      - "haystack/core/pipeline/**"
+      - "haystack/core/serialization.py"
+      - "haystack/core/errors.py"
+      - "haystack/marshal/**"
+      - "haystack/dataclasses/**"
+      - "haystack/utils/filters.py"
+      - "test/fuzz/**"
+      - ".clusterfuzzlite/**"
+      - ".github/workflows/cflite_pr.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  pr-fuzzing:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: cflite-pr-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    steps:
+      - name: Build Fuzzers
+        id: build
+        uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+        with:
+          language: python
+          sanitizer: address
+          # Needed so build_fuzzers can check out the PR base for `mode: code-change`
+          # diffing in the Run Fuzzers step below.
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Fuzzers
+        id: run
+        uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fuzz-seconds: 120
+          mode: code-change
+          sanitizer: address
+          # Crashes fail the job and are uploaded as artifacts. SARIF upload is
+          # disabled to keep the token least-privilege (no security-events: write).
+          output-sarif: false

.github/workflows/check_api_ref.yml

@@ -0,0 +1,99 @@
+name: Check API reference changes
+
+on:
+  pull_request:
+    paths:
+      - "haystack/**/*.py"
+      - "pydoc/*.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  test-api-reference-build:
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.13"
+
+      - name: Detect API reference changes
+        id: changed
+        shell: python
+        run: |
+          import os
+          import subprocess
+          from pathlib import Path
+
+          import sys
+          sys.path.insert(0, ".github/utils")
+          from docstrings_checksum import docstrings_checksum
+
+          def git(*args):
+              result = subprocess.run(["git", *args], capture_output=True, text=True)
+              return result.stdout.strip(), result.returncode
+
+          base_sha, _ = git("rev-parse", "HEAD^1")
+          diff_output, _ = git("diff", "--name-only", f"{base_sha}...HEAD")
+          changed_files = set(diff_output.splitlines())
+
+          needs_check = False
+
+          # If any pydoc config changed, always rebuild
+          if any(f.startswith("pydoc/") and f.endswith(".yml") for f in changed_files):
+              needs_check = True
+
+          # If Python files changed, compare docstring checksums
+          if not needs_check and any(f.startswith("haystack/") and f.endswith(".py") for f in changed_files):
+              runner_temp = os.environ["RUNNER_TEMP"]
+              base_worktree = os.path.join(runner_temp, "base")
+              _, rc = git("worktree", "add", base_worktree, base_sha)
+
+              pr_checksum = docstrings_checksum(Path(".").glob("haystack/**/*.py"))
+              base_checksum = ""
+              if rc == 0:
+                  base_checksum = docstrings_checksum(Path(base_worktree).glob("haystack/**/*.py"))
+
+              if pr_checksum != base_checksum:
+                  needs_check = True
+
+          print(f"API reference check needed: {needs_check}")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+              f.write(f"needs_check={str(needs_check).lower()}\n")
+
+      - name: Install Hatch
+        if: steps.changed.outputs.needs_check == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch --uploaded-prior-to=P1D
+
+      - name: Generate API references
+        if: steps.changed.outputs.needs_check == 'true'
+        run: hatch run docs
+
+      - name: Set up Node.js
+        if: steps.changed.outputs.needs_check == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Run Docusaurus md/mdx checker
+        if: steps.changed.outputs.needs_check == 'true'
+        working-directory: tmp_api_reference
+        run: |
+          # docusaurus-mdx-checker is a package that is not frequently updated. Its dependency katex sometimes ships a
+          # broken ESM build, where a __VERSION__ placeholder is left unresolved, causing a ReferenceError at import time.
+          # Node 22+ prefers ESM when available. We force CJS (CommonJS) resolution to use the working katex build.
+          # This should be safe because docusaurus-mdx-checker and its dependencies provide CJS builds.
+          export NODE_OPTIONS="--conditions=require"
+          npx docusaurus-mdx-checker@3.0.0 -v || {
+              echo ""
+              echo "For common MDX problems, see https://docusaurus.io/blog/preparing-your-site-for-docusaurus-v3#common-mdx-problems"
+              exit 1
+            }

.github/workflows/codeql.yml

@@ -0,0 +1,56 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Cancel a superseded run for the same ref (e.g. a force-push to a PR) so we
+# don't burn CI minutes analyzing commits that are already stale.
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      # Required to upload code-scanning results to the Security tab.
+      security-events: write
+      # Required by codeql-action to read the workflow/CI config.
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Haystack is a Python project; Python is interpreted, so no build is
+          # needed. Add `javascript-typescript` here if the docs-website JS code
+          # should also be scanned.
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/coverage_comment.yml

@@ -0,0 +1,24 @@
+name: Add comment about test coverage to PRs
+
+on:
+  workflow_run:
+    workflows:
+      - "Tests"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  comment:
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: py-cov-action/python-coverage-comment-action@5d8df5979747514c914e1c5a12335a7cf9a2745f # v4.1
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 60

.github/workflows/docker_release.yml

@@ -0,0 +1,88 @@
+name: Docker image release
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/docker_release.yml'
+      - 'docker/**'
+      - 'haystack/**'
+      - 'pyproject.toml'
+      - 'VERSION.txt'
+    tags:
+      - "v2.[0-9]+.[0-9]+*"
+
+env:
+  DOCKER_REPO_NAME: deepset/haystack
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    name: Build base image
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepset-ai'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Login to DockerHub
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        with:
+          images: $DOCKER_REPO_NAME
+
+      - name: Detect stable version
+        run: |
+          if [[ "${{ steps.meta.outputs.version }}" =~ ^v2\.[0-9]+\.[0-9]+$ ]]; then
+            echo "IS_STABLE=true" >> "$GITHUB_ENV"
+            echo "Stable version detected"
+          else
+            echo "Not a stable version"
+          fi
+      - name: Build base images
+        uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0
+        env:
+          IMAGE_TAG_SUFFIX: ${{ steps.meta.outputs.version }}
+          HAYSTACK_VERSION: ${{ steps.meta.outputs.version }}
+        with:
+          source: ./docker
+          targets: base
+          push: true
+
+      - name: Test base image
+        run: |
+          EXPECTED_VERSION=$(cat VERSION.txt)
+          if [[ $EXPECTED_VERSION == *"-"* ]]; then
+            EXPECTED_VERSION=$(cut -d '-' -f 1 < VERSION.txt)$(cut -d '-' -f 2 < VERSION.txt)
+          fi
+          TAG="base-${{ steps.meta.outputs.version }}"
+
+          PLATFORM="linux/amd64"
+          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
+          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"
+
+          PLATFORM="linux/arm64"
+          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
+          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"
+
+          # Remove image after test to avoid filling the GitHub runner and prevent its failure
+          docker rmi "deepset/haystack:$TAG"

.github/workflows/docs-website-test-docs-snippets.yml

@@ -0,0 +1,74 @@
+name: Test Python snippets in docs
+
+on:
+  schedule:
+    - cron: "17 3 * * *" # daily at 03:17 UTC
+  workflow_dispatch:
+    inputs:
+      haystack_version:
+        description: "Haystack version to test against (e.g., 2.16.1, main)"
+        required: false
+        default: "main"
+        type: string
+
+env:
+  HATCH_VERSION: "1.16.5"
+  PYTHON_VERSION: "3.11"
+
+permissions:
+  contents: read
+
+jobs:
+  test-docs-snippets:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+      AZURE_OPENAI_AD_TOKEN: ${{ secrets.AZURE_OPENAI_AD_TOKEN }}
+      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+      CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
+      HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: sudo apt-get install -y ffmpeg
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Generate API reference for Docusaurus
+        run: hatch run docs
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests toml --uploaded-prior-to=P1D
+
+      - name: Run snippet tests (verbose)
+        shell: bash
+        run: |
+          hatch -e test env run -- python docs-website/scripts/test_python_snippets.py --verbose tmp_api_reference/
+
+  notify-slack-on-failure:
+    if: failure() && github.ref_name == 'main'
+    needs:
+      - test-docs-snippets
+    runs-on: ubuntu-slim
+    steps:
+      - uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/docs_search_sync.yml

@@ -0,0 +1,60 @@
+name: Docs Search Sync
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 1 * * *"
+
+# Running this workflow multiple times in parallel can cause issues with files uploads and deletions.
+concurrency:
+  group: docs-search-sync
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  docs-search-sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Install Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Install Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Docusaurus and build docs-website
+        working-directory: docs-website
+        run: |
+          npm install --ignore-scripts
+          npm rebuild sharp
+          npm run build
+
+      - name: Install script dependencies
+        # sniffio is needed because of https://github.com/deepset-ai/deepset-cloud-sdk/issues/286
+        # we pin pyrate-limiter due to https://github.com/deepset-ai/deepset-cloud-sdk/issues/295
+        run: |
+          python -m pip install --upgrade pip
+          pip install deepset-cloud-sdk sniffio requests "pyrate-limiter<4" --uploaded-prior-to=P1D
+
+      - name: Update new docs to Search pipeline and remove outdated docs
+        env:
+          DEEPSET_WORKSPACE_DOCS_SEARCH: ${{ secrets.DEEPSET_WORKSPACE_DOCS_SEARCH }}
+          DEEPSET_API_KEY_DOCS_SEARCH: ${{ secrets.DEEPSET_API_KEY_DOCS_SEARCH }}
+        run: python ./.github/utils/docs_search_sync.py
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/docstring_labeler.yml

@@ -0,0 +1,70 @@
+name: Detect docstrings edit
+
+# This workflow runs in the *untrusted* pull_request context: it has a read-only
+# token and no access to secrets, so checking out the PR head here is safe.
+# It only computes whether docstrings changed and hands the result to the
+# "Apply docstrings label" workflow (workflow_run), which holds the write token
+# but never checks out untrusted code. See:
+# https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
+on:
+  pull_request:
+    paths:
+      - "haystack/**/*.py"
+
+permissions:
+  contents: read
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  detect:
+    runs-on: ubuntu-slim
+
+    steps:
+      - name: Checkout base commit
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.base_ref }}
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Get base docstrings
+        id: base-docstrings
+        run: |
+          CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
+          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout HEAD commit
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Get HEAD docstrings
+        id: head-docstrings
+        run: |
+          CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
+          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
+
+      - name: Write result
+        # The follow-up workflow reads these values; it never checks out the PR code.
+        run: |
+          mkdir -p result
+          echo "${{ github.event.pull_request.number }}" > result/pr_number
+          if [ "${{ steps.base-docstrings.outputs.checksum }}" != "${{ steps.head-docstrings.outputs.checksum }}" ]; then
+            echo "true" > result/should_label
+          else
+            echo "false" > result/should_label
+          fi
+
+      - name: Upload result
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: docstring-label-result
+          path: result/
+          retention-days: 1

.github/workflows/docstring_labeler_apply.yml

@@ -0,0 +1,42 @@
+name: Apply docstrings label
+
+# Runs after "Detect docstrings edit" completes. This workflow holds the write
+# token but only consumes the data artifact produced by the untrusted run — it
+# never checks out PR code, so it is safe under the workflow_run trigger.
+on:
+  workflow_run:
+    workflows: ["Detect docstrings edit"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  label:
+    runs-on: ubuntu-slim
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download result
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: docstring-label-result
+          path: result
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add label
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          SHOULD_LABEL=$(cat result/should_label)
+          PR_NUMBER=$(cat result/pr_number)
+          if [ "$SHOULD_LABEL" = "true" ]; then
+            gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label "type:documentation"
+          else
+            echo "Docstrings unchanged, nothing to label."
+          fi

.github/workflows/docusaurus_sync.yml

@@ -0,0 +1,66 @@
+name: Sync docs with Docusaurus
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pydoc/**"
+      - "haystack/**"
+      - ".github/workflows/docusaurus_sync.yml"
+
+env:
+  HATCH_VERSION: "1.16.5"
+  PYTHON_VERSION: "3.11"
+
+permissions:
+  contents: read
+
+jobs:
+  sync:
+    runs-on: ubuntu-slim
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Generate API reference for Docusaurus
+        run: hatch run docs
+
+      - name: Sync generated API reference to docs folder
+        run: |
+          SOURCE_PATH="tmp_api_reference"
+          DEST_PATH="docs-website/reference/haystack-api"
+
+          echo "Syncing from $SOURCE_PATH to $DEST_PATH"
+          mkdir -p $DEST_PATH
+          # Using rsync to copy files. This will also remove files in dest that are no longer in source.
+          rsync -av --delete --exclude='.git/' "$SOURCE_PATH/" "$DEST_PATH/"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Sync Haystack API reference on Docusaurus"
+          branch: sync-docusaurus-api-reference
+          base: main
+          title: "docs: sync Haystack API reference on Docusaurus"
+          add-paths: |
+            docs-website/reference/haystack-api
+          body: |
+            This PR syncs the Haystack API reference on Docusaurus. Just approve and merge it.

.github/workflows/e2e.yml

@@ -0,0 +1,52 @@
+name: end-to-end
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    paths:
+      - "e2e/**/*.py"
+      - ".github/workflows/e2e.yml"
+
+env:
+  PYTHON_VERSION: "3.10"
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  HATCH_VERSION: "1.16.5"
+  # we use HF_TOKEN instead of HF_API_TOKEN to work around a Hugging Face bug
+  # see https://github.com/deepset-ai/haystack/issues/9552
+  HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Run tests
+        run: hatch run test:e2e
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/github_release.yml

@@ -0,0 +1,104 @@
+name: Project release on Github
+
+on:
+  workflow_dispatch:  # this is useful to re-generate the release page without a new tag being pushed
+  push:
+    tags:
+      - "v2.[0-9]+.[0-9]+*"
+      # Ignore release versions tagged with -rc0 suffix
+      - "!v2.[0-9]+.[0-9]-rc0"
+
+permissions:
+  contents: read
+
+jobs:
+  generate-notes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # ncipollo/release-action creates the GitHub release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-tags: true
+          fetch-depth: 0  # slow but needed by reno
+
+      - name: Parse version
+        id: version
+        run: |
+          echo "current_release=$(awk -F \\- '{print $1}' < VERSION.txt)" >> "$GITHUB_OUTPUT"
+          echo "current_pre_release=$(awk -F \\- '{print $2}' < VERSION.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Install reno
+        run: |
+          python -m pip install --upgrade pip
+          pip install "reno<5" --uploaded-prior-to=P1D
+
+      # Remove next version rc0 tag in the CI environment to prevent reno from assigning notes to future releases.
+      # This ensures release notes are correctly aggregated for the current version.
+      # This is a workaround. Can be removed if the release process is fully aligned with reno.
+      - name: Delete next version rc0 tag in the CI environment
+        run: |
+          # Parse version X.Y.Z and increment Y for next minor version
+          IFS='.' read -r MAJOR MINOR _ <<< "${{ steps.version.outputs.current_release }}"
+          NEXT_MINOR=$((MINOR + 1))
+          NEXT_TAG="v${MAJOR}.${NEXT_MINOR}.0-rc0"
+
+          if git rev-parse --verify "$NEXT_TAG" >/dev/null 2>&1; then
+            git tag -d "$NEXT_TAG"
+            echo "Deleted local tag $NEXT_TAG"
+          else
+            echo "Tag $NEXT_TAG does not exist locally"
+          fi
+
+      - name: Generate release notes
+        env:
+
+          EARLIEST_VERSION: v${{ steps.version.outputs.current_release }}-rc1
+        run: |
+          reno report --no-show-source --ignore-cache --earliest-version "$EARLIEST_VERSION" -o relnotes.rst
+
+      - name: Convert to Markdown
+        uses: docker://pandoc/core:3.8@sha256:c7127705b9d84c1b4acbbe411a86e3a6d67ac57870654dc378b6fe636ab747df
+        with:
+          args: "--from rst --to gfm --syntax-highlighting=none --wrap=none relnotes.rst -o relnotes.md"
+
+      # We copy the relnotes file since the original one cannot be modified due to permissions
+      - name: Copy relnotes file
+        run: |
+          cat relnotes.md > enhanced_relnotes.md
+
+      - name: Add contributor list
+          # only for minor releases and minor release candidates (not bugfix releases)
+        if: endsWith(steps.version.outputs.current_release, '.0')
+        env:
+          GH_TOKEN: ${{ github.token }}
+          START: v${{ steps.version.outputs.current_release }}-rc0
+          END: ${{ github.ref_name }}
+        run: |
+          JQ_EXPR='[.commits[].author.login]
+            | map(select(. != null and . != "HaystackBot" and . != "dependabot[bot]" and . != "github-actions[bot]"))
+            | unique
+            | sort_by(ascii_downcase)
+            | map("@\(.)")
+            | join(", ")'
+          CONTRIBUTORS=$(gh api "repos/deepset-ai/haystack/compare/$START...$END" \
+            --jq "$JQ_EXPR") || { echo "Unable to fetch contributors"; exit 1; }
+
+          {
+            echo ""
+            echo "## 💙 Big thank you to everyone who contributed to this release!"
+            echo ""
+            echo "$CONTRIBUTORS"
+          } >> enhanced_relnotes.md
+
+      - name: Debug
+        run: |
+          cat enhanced_relnotes.md
+
+      - uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0
+        with:
+          bodyFile: "enhanced_relnotes.md"
+          prerelease: ${{ steps.version.outputs.current_pre_release != '' }}
+          allowUpdates: true

.github/workflows/labeler.yml

@@ -0,0 +1,17 @@
+name: "Labeler"
+on:
+- pull_request_target
+
+permissions:
+  contents: read
+
+jobs:
+  triage:
+    runs-on: ubuntu-slim
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+    - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/license_compliance.yml

@@ -0,0 +1,69 @@
+name: License Compliance
+
+on:
+  pull_request:
+    paths:
+      - "**/pyproject.toml"
+      - ".github/workflows/license_compliance.yml"
+  # Since we test PRs, there is no need to run the workflow at each
+  # merge on `main`. Let's use a cron job instead.
+  schedule:
+    - cron: "0 0 * * *" # every day at midnight
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  license_check_direct:
+    name: Direct dependencies only
+    env:
+      REQUIREMENTS_FILE: requirements_direct.txt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Get direct dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install toml --uploaded-prior-to=P1D
+          python .github/utils/pyproject_to_requirements.py pyproject.toml > ${{ env.REQUIREMENTS_FILE }}
+
+      - name: Check Licenses
+        id: license_check_report
+        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          requirements: ${{ env.REQUIREMENTS_FILE }}
+          fail: "Copyleft,Other,Error"
+          # Exclusions in the vanilla distribution must be explicitly motivated
+          # - tqdm is MLP but there are no better alternatives
+          # - typing_extensions>=4.13.0 has a Python Software Foundation License 2.0 but pip-license-checker does not recognize it
+          #   (https://github.com/pilosus/pip-license-checker/issues/143)
+          exclude: "(?i)^(tqdm|typing_extensions).*"
+
+      # We keep the license inventory on FOSSA
+      - name: Send license report to Fossa
+        uses: fossas/fossa-action@29693cc50323968e039056be419b32989fc5880c # v2.0.0
+        continue-on-error: true # not critical
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+      - name: Print report
+        if: ${{ always() }}
+        run: echo "${{ steps.license_check_report.outputs.report }}"
+
+      - name: Notify Slack on failure
+        if: failure()
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/nightly_testpypi_release.yml

@@ -0,0 +1,51 @@
+name: Nightly pre-release on PyPI
+
+on:
+  schedule:
+    # Run at midnight UTC every day
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  nightly-release:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    # Always build from main for consistency (scheduled and manual runs)
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          persist-credentials: false
+          ref: main
+          fetch-depth: 1
+
+      # Reads VERSION.txt, strips any -rcN suffix, and appends .devYYYYMMDDHHMMSS
+      # (e.g. 2.25.0.dev20250217000000) so each run gets a unique, PEP 440–valid pre-release version.
+      - name: Set nightly version
+        id: set-version
+        run: |
+          BASE_VERSION=$(sed 's/-rc[0-9]*$//' VERSION.txt)
+          TIMESTAMP=$(date +%Y%m%d%H%M%S)
+          NIGHTLY_VERSION="${BASE_VERSION}.dev${TIMESTAMP}"
+          echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "${NIGHTLY_VERSION}" > VERSION.txt
+          echo "Building haystack-ai version: ${NIGHTLY_VERSION}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Build Haystack
+        run: hatch build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

.github/workflows/project.yml

@@ -0,0 +1,19 @@
+name: Track issues with Github project
+
+on:
+  issues:
+    types:
+      - opened
+
+permissions:
+  contents: read
+
+jobs:
+  add-to-project:
+    name: Add new issues to project for triage
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
+        with:
+          project-url: https://github.com/orgs/deepset-ai/projects/5
+          github-token: ${{ secrets.GH_PROJECT_PAT }}

.github/workflows/promote_unstable_docs.yml

@@ -0,0 +1,65 @@
+name: Release new minor version docs
+
+on:
+  push:
+    tags:
+      # Trigger this only for new minor version tags (e.g. v2.99.0)
+      - "v[0-9]+.[0-9]+.0"
+      # Exclude 1.x tags
+      - "!v1.[0-9]+.[0-9]+"
+
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  promote:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout this repo
+        # use VERSION.txt file from main branch
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: main
+
+      - name: Get version to release
+        id: version
+        shell: bash
+        # We only need `major.minor`. At this point, VERSION.txt contains the next version.
+        # For example, if we are releasing 2.20.0, VERSION.txt contains 2.21.0-rc0.
+        run: |
+          MAJOR=$(cut -d "." -f 1 < VERSION.txt)
+          MINOR=$(cut -d "." -f 2 < VERSION.txt)
+          MINOR=$((MINOR - 1))
+          echo "version=${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Promote unstable docs for Docusaurus
+        run: |
+          python ./.github/utils/promote_unstable_docs_docusaurus.py --version ${{ steps.version.outputs.version }}
+
+      - name: Create Pull Request with Docusaurus docs updates
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          branch: promote-unstable-docs-${{ steps.version.outputs.version }}
+          base: main
+          title: "docs: promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          add-paths: |
+            docs-website
+          body: |
+            This PR promotes the unstable docs for Haystack ${{ steps.version.outputs.version }} to stable.
+            It is expected to run at the time of the release.
+            You can inspect the docs preview and merge it. There should now be only one unstable version representing the next (main) branch.
+          # This workflow is triggered by a tag pushed by the HaystackBot in release.yml > create-release-tag.
+          # GitHub requires reviewers to be different from the PR author, so setting `github.actor`
+          # would fail (it would request a review from the HaystackBot itself).
+          # So we don't set any reviewers and instead notify the Release Manager
+          # (see .github/utils/prepare_release_notification.sh).

.github/workflows/push_release_notes_to_website.yml

@@ -0,0 +1,65 @@
+name: Push release notes to website
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Haystack version (vX.Y.Z)'
+        required: true
+        type: string
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  VERSION: ${{ inputs.version }}
+
+permissions:
+  contents: read
+
+jobs:
+  push-release-notes-to-website:
+
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout Haystack home repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/haystack-home
+
+      - name: Get release notes and add frontmatter
+        id: release_notes
+
+        run: |
+          VERSION_NUMBER="${VERSION:1}"
+          RELEASE_DATE=$(gh release view "$VERSION" --repo deepset-ai/haystack --json publishedAt --jq '.publishedAt | split("T")[0]')
+          RELEASE_NOTES_PATH="content/release-notes/$VERSION_NUMBER.md"
+
+          {
+            echo "---"
+            echo "title: Haystack $VERSION_NUMBER"
+            echo "description: Release notes for Haystack $VERSION_NUMBER"
+            echo "toc: True"
+            echo "date: $RELEASE_DATE"
+            echo "last_updated: $RELEASE_DATE"
+            echo 'tags: ["Release Notes"]'
+            echo "link: https://github.com/deepset-ai/haystack/releases/tag/$VERSION"
+            echo "---"
+            echo ""
+          } > "$RELEASE_NOTES_PATH"
+
+          gh release view "$VERSION" --repo deepset-ai/haystack --json body --jq '.body' >> "$RELEASE_NOTES_PATH"
+
+          cat "$RELEASE_NOTES_PATH"
+
+      - name: Create Pull Request to Haystack Home
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Add release notes for Haystack ${{ env.VERSION }}"
+          branch: add-release-notes-for-haystack-${{ env.VERSION }}
+          base: main
+          title: "docs: add release notes for Haystack ${{ env.VERSION }}"
+          add-paths: |
+            content/release-notes
+          body: |
+            This PR adds the release notes for Haystack ${{ env.VERSION }} to the website.
+          reviewers: "${{ github.actor }}"

.github/workflows/pypi_release.yml

@@ -0,0 +1,38 @@
+name: Project release on PyPi
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+      # We must not release versions tagged with -rc0 suffix
+      - "!v[0-9]+.[0-9]+.[0-9]-rc0"
+
+env:
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  release-on-pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Build Haystack
+        run: hatch build
+
+      - name: Publish on PyPi
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

.github/workflows/release.yml

@@ -0,0 +1,302 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., v2.99.0-rc1 or v2.99.0)'
+        required: true
+        type: string
+
+# Only one release workflow runs at a time; additional runs are queued.
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  authorize:
+    # Releasing acts as HaystackBot (a ruleset-bypass actor), so the ruleset
+    # cannot enforce *who* started the release. Gate on the triggering user's
+    # role instead: only maintainers/admins may run this workflow, even though
+    # workflow_dispatch itself is available to anyone with write access.
+    runs-on: ubuntu-slim
+    steps:
+      - name: Verify the triggering user may release
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          ACTOR: ${{ github.triggering_actor }}
+        run: |
+          ROLE=$(gh api "repos/${{ github.repository }}/collaborators/${ACTOR}/permission" --jq '.role_name')
+          echo "::notice::${ACTOR} has repository role '${ROLE}'"
+          case "$ROLE" in
+            admin|maintain)
+              echo "✅ ${ACTOR} is authorized to trigger a release." ;;
+            *)
+              echo "::error::${ACTOR} has role '${ROLE}'. A release can only be triggered by a user with the 'maintain' or 'admin' role."
+              exit 1 ;;
+          esac
+
+  parse-validate-version:
+    needs: ["authorize"]
+    runs-on: ubuntu-slim
+    outputs:
+      version: ${{ steps.parse-validate.outputs.version }}
+      major_minor: ${{ steps.parse-validate.outputs.major_minor }}
+      release_branch: ${{ steps.parse-validate.outputs.release_branch }}
+      is_rc: ${{ steps.parse-validate.outputs.is_rc }}
+      is_first_rc: ${{ steps.parse-validate.outputs.is_first_rc }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+      - name: Parse and validate version
+        id: parse-validate
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ inputs.version }}
+        run: .github/utils/parse_validate_version.sh "$VERSION"
+
+  branch-off:
+    needs: ["parse-validate-version"]
+    if: needs.parse-validate-version.outputs.is_first_rc == 'true'
+    uses: ./.github/workflows/branch_off.yml
+    # https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows#passing-secrets-to-nested-workflows
+    secrets: inherit
+
+  create-release-tag:
+    needs: ["parse-validate-version", "branch-off"]
+    if: always() && needs.parse-validate-version.result == 'success' && needs.branch-off.result != 'failure'
+    runs-on: ubuntu-slim
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+          # use this token so the created tag triggers workflows (does not happen with the default github.token)
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+      - name: Update VERSION.txt and create tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          git checkout ${{ needs.parse-validate-version.outputs.release_branch }}
+          git pull origin ${{ needs.parse-validate-version.outputs.release_branch }}
+
+          echo "${{ needs.parse-validate-version.outputs.version }}" > VERSION.txt
+          git add VERSION.txt
+          git commit -m "bump version to ${{ needs.parse-validate-version.outputs.version }}"
+          git push origin ${{ needs.parse-validate-version.outputs.release_branch }}
+
+          TAG="v${{ needs.parse-validate-version.outputs.version }}"
+          git tag -m "$TAG" "$TAG"
+          git push origin "$TAG"
+
+  check-artifacts:
+    needs: ["parse-validate-version", "create-release-tag"]
+    if: always() && needs.parse-validate-version.result == 'success' && needs.create-release-tag.result == 'success'
+    runs-on: ubuntu-latest
+    outputs:
+      github_url: ${{ steps.set-outputs.outputs.github_url }}
+      pypi_url: ${{ steps.set-outputs.outputs.pypi_url }}
+      docker_url: ${{ steps.set-outputs.outputs.docker_url }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+
+      - name: Wait for release workflows
+        run: |
+          .github/utils/wait_for_workflows.sh "v${{ env.VERSION }}" \
+            "Project release on PyPi" \
+            "Project release on Github" \
+            "Docker image release"
+
+      - name: Check artifacts
+        run: |
+          check() {
+            for _ in {1..5}; do curl -sf "$2" > /dev/null && echo "✅ $1" && return 0; sleep 30; done
+            echo "❌ $1 not found" && return 1
+          }
+          check "GitHub Release" "https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.VERSION }}"
+          check "PyPI package" "https://pypi.org/pypi/haystack-ai/${{ env.VERSION }}/json"
+          check "Docker image" "https://hub.docker.com/v2/repositories/deepset/haystack/tags/base-v${{ env.VERSION }}"
+
+      - name: Set artifact URLs
+        id: set-outputs
+        run: |
+          {
+            echo "github_url=https://github.com/${{ github.repository }}/releases/tag/v${{ env.VERSION }}"
+            echo "pypi_url=https://pypi.org/project/haystack-ai/${{ env.VERSION }}/"
+            echo "docker_url=https://hub.docker.com/r/deepset/haystack/tags?name=base-v${{ env.VERSION }}"
+          } >> "$GITHUB_OUTPUT"
+
+  bump-dc-pipeline-templates:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout dc-pipeline-templates
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/dc-pipeline-templates
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Update haystack pin
+        run: sed -i "s/haystack-ai>=.*/haystack-ai>=${VERSION}/" requirements-test.txt
+
+      - name: Create Pull Request
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Bump haystack to ${{ env.VERSION }}"
+          branch: bump-haystack-${{ env.VERSION }}
+          base: main
+          title: "chore: bump haystack to ${{ env.VERSION }}"
+          add-paths: requirements-test.txt
+          body: |
+            Bump haystack pin to `${{ env.VERSION }}` for platform testing.
+
+  bump-deepset-cloud-custom-nodes:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout haystack (for utils)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          path: haystack
+          sparse-checkout: .github/utils/update_haystack_dc_custom_nodes.py
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout deepset-cloud-custom-nodes
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/deepset-cloud-custom-nodes
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          path: deepset-cloud-custom-nodes
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.13"
+
+      - name: Install tomlkit
+        run: |
+          python -m pip install --upgrade pip
+          pip install tomlkit --uploaded-prior-to=P1D
+
+      - name: Update haystack-ai in uv.lock
+        run: python haystack/.github/utils/update_haystack_dc_custom_nodes.py "${{ env.VERSION }}" deepset-cloud-custom-nodes/uv.lock
+
+      - name: Create Pull Request
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          path: deepset-cloud-custom-nodes
+          commit-message: "Bump haystack to ${{ env.VERSION }}"
+          branch: bump-haystack-${{ env.VERSION }}
+          base: main
+          title: "chore: bump haystack to ${{ env.VERSION }}"
+          add-paths: uv.lock
+          body: |
+            Bump haystack pin to `${{ env.VERSION }}` for platform testing.
+
+  bump-haystack-runtime:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.wait-pr.outputs.pr_url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Trigger "Update package version" workflow
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          gh workflow run update-package-version.yaml \
+            -R deepset-ai/haystack-runtime \
+            -f haystack_version="${{ env.VERSION }}"
+
+      - name: Wait for PR
+        id: wait-pr
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          for i in $(seq 1 30); do
+            PR_URL=$(gh pr list -R deepset-ai/haystack-runtime \
+              --head "bump/hs${{ env.VERSION }}" \
+              --json url,createdAt \
+              --jq ".[] | select(.createdAt >= \"$TRIGGER_TIME\") | .url")
+            if [[ -n "$PR_URL" ]]; then
+              echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+              echo "Found PR: $PR_URL"
+              exit 0
+            fi
+            echo "Attempt $i: PR not found yet, waiting 10s..."
+            sleep 10
+          done
+          echo "PR not found after 5 minutes"
+
+  notify:
+    needs:
+      - "parse-validate-version"
+      - "branch-off"
+      - "create-release-tag"
+      - "check-artifacts"
+      - "bump-dc-pipeline-templates"
+      - "bump-deepset-cloud-custom-nodes"
+      - "bump-haystack-runtime"
+    if: always()
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Prepare release notification
+        env:
+          VERSION: ${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          HAS_FAILURE: ${{ contains(needs.*.result, 'failure') }}
+          IS_RC: ${{ needs.parse-validate-version.outputs.is_rc }}
+          IS_FIRST_RC: ${{ needs.parse-validate-version.outputs.is_first_rc }}
+          MAJOR_MINOR: ${{ needs.parse-validate-version.outputs.major_minor }}
+          GITHUB_URL: ${{ needs.check-artifacts.outputs.github_url }}
+          PYPI_URL: ${{ needs.check-artifacts.outputs.pypi_url }}
+          DOCKER_URL: ${{ needs.check-artifacts.outputs.docker_url }}
+          BUMP_VERSION_PR_URL: ${{ needs.branch-off.outputs.bump_version_pr_url }}
+          DC_PIPELINE_TEMPLATES_PR_URL: ${{ needs.bump-dc-pipeline-templates.outputs.pr_url }}
+          DC_CUSTOM_NODES_PR_URL: ${{ needs.bump-deepset-cloud-custom-nodes.outputs.pr_url }}
+          HAYSTACK_RUNTIME_PR_URL: ${{ needs.bump-haystack-runtime.outputs.pr_url }}
+        run: .github/utils/prepare_release_notification.sh
+
+      - name: Send release notification to Slack
+        uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }}
+          webhook-type: incoming-webhook
+          payload-file-path: slack_payload.json

.github/workflows/release_notes.yml

@@ -0,0 +1,85 @@
+name: Check Release Notes
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+      - labeled
+      - unlabeled
+    paths:
+      - "**.py"
+      - "pyproject.toml"
+      - "!.github/**/*.py"
+      - "releasenotes/notes/*.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  reno:
+    runs-on: ubuntu-slim
+    env:
+      PYTHON_VERSION: "3.10"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+      - name: Get release note files
+        id: changed-files
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
+        with:
+          files: releasenotes/notes/*.yaml
+
+      - name: Check release notes
+        if: steps.changed-files.outputs.any_changed == 'false' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        run: |
+          # Check if any of the commit messages contain tags ci/docs/test
+          if git log --pretty=%s origin/main..HEAD | grep -E '^(ci:|docs:|test:)' > /dev/null; then
+            echo "Skipping release note check for commits with 'ci:', 'docs:', or 'test:' tags."
+          else
+            echo "::error::The release notes file is missing, please add one or attach the label 'ignore-for-release-notes' to this PR."
+            exit 1
+          fi
+
+      - name: Verify release notes formatting
+        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        run: |
+          python -m pip install --upgrade pip
+          pip install "reno<5" --uploaded-prior-to=P1D
+          reno lint .  # it is not possible to pass a list of files to reno lint
+
+      - name: Check reStructuredText code formatting
+        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        shell: python
+        run: |
+          files = "${{ steps.changed-files.outputs.all_changed_files }}".split()
+          errors = []
+
+          for filepath in files:
+            with open(filepath) as f:
+              for line_no, line in enumerate(f, start=1):
+                # Check for triple backticks (Markdown code blocks)
+                if "```" in line:
+                  err = (f"Format error in {filepath}:{line_no}: "
+                         "Found triple backticks. Use reStructuredText code block directive instead: .. code:: python")
+                  errors.append(err)
+
+                # Check for single backticks (Markdown inline code)
+                if "`" in line.replace("```", "").replace("``", ""):
+                  err = (f"Format error in {filepath}:{line_no}: "
+                         "Found single backticks. Use double backticks (``code``) for inline code.")
+                  errors.append(err)
+
+          if errors:
+              raise Exception("\n".join(errors))

.github/workflows/release_notes_skipper.yml

@@ -0,0 +1,26 @@
+name: Check Release Notes
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+      - labeled
+      - unlabeled
+    paths-ignore:
+      - "**.py"
+      - "pyproject.toml"
+      - "!.github/**/*.py"
+      - "releasenotes/notes/*.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  reno:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Skip mandatory job
+        run: echo "Skipped!"

.github/workflows/scorecard.yml

@@ -0,0 +1,66 @@
+# OpenSSF Scorecard
+#
+# Runs the OpenSSF Scorecard analyzer (https://github.com/ossf/scorecard) and
+# publishes results to GitHub's Security tab via the code-scanning SARIF API.
+#
+# Why: Scorecard gives haystack maintainers a single weekly signal on
+# supply-chain posture (branch protection, SAST coverage, dependency-update
+# tool, signed releases, etc.) without changing any existing CI behaviour.
+# It also lets downstream consumers verify the project's score on the
+# OpenSSF deps.dev / Security Insights dashboards.
+#
+# Permissions are scoped to the minimum required by the action:
+#   - security-events: write   -> upload SARIF
+#   - id-token:        write   -> publish to the public OpenSSF API (badge)
+#   - contents:        read    -> default repo read
+#   - actions:         read    -> read CI config for checks
+#
+# Default branch is 'main'.
+
+name: OpenSSF Scorecard
+
+on:
+  # Run weekly so results stay fresh, plus on branch-protection changes so the
+  # branch-protection check is re-evaluated immediately in the Security tab.
+  branch_protection_rule:
+  schedule:
+    - cron: '32 6 * * 1'
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+      contents: read
+      actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Run analysis
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # Publish results to the OpenSSF public REST API so the badge in
+          # the README and deps.dev card stay up to date. Opt-in.
+          publish_results: true
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      - name: Upload to code-scanning
+        uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          sarif_file: results.sarif

.github/workflows/slow.yml

@@ -0,0 +1,130 @@
+name: Slow Integration Tests
+
+# The workflow will always run, but the actual tests will only execute when:
+# - The workflow is triggered manually
+# - The workflow is scheduled
+# - The PR has the "run-slow-tests" label
+# - The push is to a release branch
+# - There are changes to relevant files.
+# Note: If no conditions are met, the workflow will complete successfully without running tests
+# to satisfy Branch Protection rules.
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+  PYTHON_VERSION: "3.10"
+  HATCH_VERSION: "1.16.5"
+  HAYSTACK_MPS_ENABLED: false
+  HAYSTACK_XPU_ENABLED: false
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 0 * * *"
+  push:
+    branches:
+      # release branches have the form v1.9.x
+      - "v[0-9].*[0-9].x"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+      - unlabeled
+
+permissions:
+  contents: read
+
+jobs:
+  check-if-changed:
+    # This job checks if the relevant files have been changed.
+    # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
+    # This ensures the "Slow Integration Tests completed" job always runs, which is required by Branch Protection rules.
+    name: Check if changed
+    runs-on: ubuntu-slim
+    permissions:
+      pull-requests: read
+    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
+    outputs:
+      changes: ${{ steps.changes.outputs.changes }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Check for changed code
+        id: changes
+        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        with:
+          # List of Python files that trigger slow integration tests when modified
+          filters: |
+            changes:
+              - "haystack/components/evaluators/sas_evaluator.py"
+              - "haystack/components/generators/openai_dalle.py"
+              - "haystack/components/preprocessors/embedding_based_document_splitter.py"
+              - "haystack/components/retrievers/multi_query_embedding_retriever.py"
+
+              - "test/components/evaluators/test_sas_evaluator.py"
+              - "test/components/generators/test_openai_dalle.py"
+              - "test/components/preprocessors/test_embedding_based_document_splitter.py"
+              - "test/components/retrievers/test_multi_query_embedding_retriever.py"
+
+  slow-integration-tests:
+    name: Slow Tests / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: check-if-changed
+    timeout-minutes: 30
+    # Run tests if: manual trigger, scheduled, PR has label, release branch, or relevant files changed
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'schedule' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests')) ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/v[0-9].*[0-9].x') ||
+      (needs.check-if-changed.outputs.changes == 'true')
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Run tests
+        run: hatch run test:integration-only-slow
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}
+
+  slow-integration-tests-completed:
+    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
+    name: Slow Integration Tests completed
+    runs-on: ubuntu-slim
+    if: ${{ always() && !cancelled() }}
+    needs: slow-integration-tests
+
+    steps:
+      - name: Mark tests as completed
+        run: |
+          if [ "${{ needs.slow-integration-tests.result }}" = "failure" ]; then
+            echo "Slow Integration Tests failed!"
+            exit 1
+          else
+            echo "Slow Integration Tests completed!"
+          fi

.github/workflows/stale.yml

@@ -0,0 +1,21 @@
+name: 'Stalebot'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  makestale:
+    runs-on: ubuntu-slim
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
+        with:
+          any-of-labels: 'proposal,information-needed'
+          stale-pr-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
+          days-before-stale: 30
+          days-before-close: 10

.github/workflows/tests.yml

@@ -0,0 +1,343 @@
+name: Tests
+
+# The workflow will always run, but the actual tests will only execute when:
+# - The workflow is triggered manually.
+# - The push is to main or a release branch.
+# - There are changes to relevant files on a pull request.
+# Note: If no conditions are met, the workflow will complete successfully without running tests
+# to satisfy Branch Protection rules.
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  push:
+    branches:
+      - main
+      # release branches have the form v1.9.x
+      - "v[0-9].*[0-9].x"
+    # when we push, we do not need to satisfy Branch Protection rules, so we can ignore PRs that just change docs
+    paths-ignore:
+      - "docs/**"
+      - "docs-website/**"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
+  CORE_AZURE_CS_API_KEY: ${{ secrets.CORE_AZURE_CS_API_KEY }}
+  AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+  AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+  PYTHON_VERSION: "3.10"
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  check-if-changed:
+    # This job checks if the relevant files have been changed.
+    # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
+    # This ensures the "Mark tests as completed" job always runs, which is required by Branch Protection rules.
+    name: Check if changed
+    runs-on: ubuntu-slim
+    permissions:
+      pull-requests: read
+    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
+    outputs:
+      changes: ${{ steps.changes.outputs.changes }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Check for changed code
+        id: changes
+        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        with:
+          filters: |
+            changes:
+              - "haystack/**/*.py"
+              - "test/**/*.py"
+              - "pyproject.toml"
+              - ".github/utils/*.py"
+              - "scripts/*.py"
+
+  format:
+    needs: check-if-changed
+    # Run tests if: manual trigger, push to main/release, or relevant files changed
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'push' ||
+      (needs.check-if-changed.outputs.changes == 'true')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Ruff - check format and linting
+        run: hatch run fmt-check
+
+      - name: Check presence of license header
+        run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye:v6.5.1@sha256:0ebd72353053aa6d3f0cb44a6897786386347bd4636ffc9592f8e47c81ea2b9d check
+
+  check-imports:
+    needs: format
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Check imports
+        run: hatch run python .github/utils/check_imports.py
+
+  unit-tests:
+    name: Unit / ${{ matrix.os }}
+    needs: format
+    timeout-minutes: 30
+    permissions:
+      contents: write
+      pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: true # needed for python-coverage-comment-action
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:unit
+
+      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: cache
+        if: matrix.os == 'macos-latest'
+        with:
+          path: ${{ steps.hatch.outputs.env }}
+          key: ${{ runner.os }}-${{ github.sha }}
+
+      # On PR: posts coverage comment (directly on same-repo PRs; via artifact for fork PRs).
+      # On push to main: stores coverage baseline on the data branch.
+      # We only upload coverage for ubuntu-latest, as handling multiple OSes adds complexity for little gain.
+      - name: Coverage comment
+        id: coverage_comment
+        if: matrix.os == 'ubuntu-latest'
+        uses: py-cov-action/python-coverage-comment-action@5d8df5979747514c914e1c5a12335a7cf9a2745f # v4.1
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 60
+
+      - name: Upload coverage comment to be posted
+        if: matrix.os == 'ubuntu-latest' && github.event_name == 'pull_request' && steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt
+
+  mypy:
+    needs: unit-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+      - name: Get changed files
+        id: files
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
+        with:
+          files: |
+            **/*.py
+            pyproject.toml
+          files_ignore: |
+            test/**
+            .github/**
+            scripts/**
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        if: steps.files.outputs.any_changed == 'true'
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Mypy
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          mkdir .mypy_cache
+          hatch run test:types
+
+  integration-tests-linux:
+    name: Integration / ubuntu-latest
+    needs: unit-tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  integration-tests-macos:
+    name: Integration / macos-latest
+    needs: unit-tests
+    runs-on: macos-latest
+    timeout-minutes: 30
+    env:
+      HAYSTACK_MPS_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: cache
+        with:
+          path: ${{ steps.hatch.outputs.env }}
+          key: ${{ runner.os }}-${{ github.sha }}
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  integration-tests-windows:
+    name: Integration / windows-latest
+    needs: unit-tests
+    runs-on: windows-latest
+    timeout-minutes: 30
+    env:
+      HAYSTACK_XPU_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  notify-slack-on-failure:
+    if: failure() && github.ref_name == 'main'
+    needs:
+      - check-imports
+      - mypy
+      - integration-tests-linux
+      - integration-tests-macos
+      - integration-tests-windows
+    runs-on: ubuntu-slim
+    steps:
+      - uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}
+
+  tests-completed:
+    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
+    name: Mark tests as completed
+    runs-on: ubuntu-slim
+    if: ${{ always() && !cancelled() }}
+    needs:
+      - check-imports
+      - mypy
+      - integration-tests-linux
+      - integration-tests-macos
+      - integration-tests-windows
+
+    steps:
+      - name: Mark tests as completed
+        run: |
+          if [ "${{ needs.check-imports.result }}" = "failure" ] ||
+             [ "${{ needs.mypy.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-linux.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-macos.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-windows.result }}" = "failure" ]; then
+            echo "Tests failed!"
+            exit 1
+          else
+            echo "Tests completed!"
+          fi

.github/workflows/update-platform-components.yml

@@ -0,0 +1,99 @@
+name: Update Platform Components Table
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: update-platform-components
+  cancel-in-progress: false
+
+env:
+  PYTHON_VERSION: "3.11"
+  OUTPUT_FILE: docs-website/docs/overview/platform-components.mdx
+
+permissions:
+  contents: read
+
+jobs:
+  update:
+    runs-on: ubuntu-slim
+
+    steps:
+      - name: Checkout deepset-ai/haystack
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          # Full history so git diff against HEAD works correctly
+          fetch-depth: 0
+
+      - name: Checkout deepset-ai/haystack-core-integrations
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/haystack-core-integrations
+          path: haystack-core-integrations
+          fetch-depth: 1
+          sparse-checkout: integrations
+          sparse-checkout-cone-mode: true
+
+      - name: Checkout platform repo (sparse — schema file only)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: ${{ secrets.SCHEMA_REPO }}
+          token: ${{ secrets.PLATFORM_REPO_PAT }}
+          path: platform-repo
+          sparse-checkout: |
+            ${{ secrets.SCHEMA_REPO_PATH }}
+          sparse-checkout-cone-mode: false
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Generate platform components table
+        run: |
+          python scripts/generate_platform_components_table.py \
+            --haystack-src . \
+            --integrations-src ./haystack-core-integrations \
+            --schema ./platform-repo/${{ secrets.SCHEMA_REPO_PATH }} \
+            --output ./${{ env.OUTPUT_FILE }}
+
+      - name: Compute PR body
+        id: pr_body
+        run: |
+          FILE="${{ env.OUTPUT_FILE }}"
+          if git ls-files --error-unmatch "$FILE" 2>/dev/null; then
+            ADDED=$(git diff HEAD -- "$FILE" | grep -cE '^\+\| `' || true)
+            REMOVED=$(git diff HEAD -- "$FILE" | grep -cE '^\-\| `' || true)
+            SUMMARY="Component rows changed: **+${ADDED}** added, **-${REMOVED}** removed."
+          else
+            SUMMARY="Initial generation of the platform components table."
+          fi
+
+          {
+            echo "body<<EOF"
+            echo "## Platform Components Table Update"
+            echo ""
+            echo "${SUMMARY}"
+            echo ""
+            echo "_Auto-generated by the [update-platform-components](https://github.com/deepset-ai/haystack/actions/workflows/update-platform-components.yml) workflow._"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Open or update pull request
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "chore: update platform components table"
+          branch: chore/update-platform-components
+          base: main
+          title: "chore: update platform components table"
+          add-paths: |
+            ${{ env.OUTPUT_FILE }}
+          body: ${{ steps.pr_body.outputs.body }}
+          labels: |
+            documentation
+            ignore-for-release-notes

.github/workflows/workflows_linting.yml

@@ -0,0 +1,30 @@
+name: Github workflows linter
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/**"
+
+permissions:
+  contents: read
+
+jobs:
+  lint-workflows:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: ">=1.24.0"
+
+      - name: Install actionlint
+        run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12
+
+      - name: Run actionlint
+        env:
+          SHELLCHECK_OPTS: --exclude=SC2102
+        run: actionlint

.gitignore

@@ -0,0 +1,178 @@
+# Local run files
+qa.db
+**/qa.db
+**/*qa*.db
+**/test-reports
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Docs website (Docusaurus)
+docs-website/.docusaurus/
+docs-website/build/
+docs-website/node_modules/
+docs-website/.cache-loader/
+docs-website/.eslintcache
+docs-website/.netlify/
+docs-website/coverage/
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# API reference
+pydoc/temp/
+tmp_api_reference/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# PyCharm
+.idea
+
+# VSCode
+.vscode
+
+# haystack files
+haystack/document_store/qa.db
+**/mlruns/**
+src
+!docs-website/src/
+models
+saved_models
+*_build
+rest_api/file-upload/*
+**/feedback_squad_direct.json
+haystack/json-schemas
+.haystack_debug
+
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache
+
+# Zed configs
+.zed/*
+
+# uv
+uv.lock

.pre-commit-config.yaml

@@ -0,0 +1,57 @@
+fail_fast: true
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-ast # checks Python syntax
+      - id: check-json # checks JSON syntax
+      - id: check-merge-conflict # checks for no merge conflict strings
+      - id: check-shebang-scripts-are-executable # checks all shell scripts have executable permissions
+      - id: check-toml # checks TOML syntax
+      - id: check-yaml # checks YAML syntax
+      - id: end-of-file-fixer # checks there is a newline at the end of the file
+      - id: mixed-line-ending # normalizes line endings
+      - id: no-commit-to-branch # prevents committing to main
+      - id: trailing-whitespace # trims trailing whitespace
+        args: [--markdown-linebreak-ext=md, --markdown-linebreak-ext=mdx]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.2
+    hooks:
+    - id: ruff-check
+      args: [ --fix ]
+    - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: ruff-format-docs
+        name: ruff-format-docs
+        language: python
+        entry: python scripts/ruff_format_docs.py --line-length=88
+        files: ^docs-website/.*\.mdx$
+        types: [text]
+        additional_dependencies:
+          - ruff
+          - add-trailing-comma
+
+      - id: release-note-backticks
+        name: release-note-backticks
+        language: python
+        entry: python scripts/release_note_backticks.py
+        files: ^releasenotes/notes/.*\.yaml$
+        types: [text]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        args: ["--toml", "pyproject.toml"]
+        additional_dependencies:
+          - tomli
+
+  - repo: https://github.com/rhysd/actionlint
+    rev: v1.7.10
+    hooks:
+      - id: actionlint-docker
+        args: ["-ignore", "SC2102"]

AGENTS.md

@@ -0,0 +1,59 @@
+# Haystack Guidelines for AI Agents
+
+## Environment
+
+Haystack uses **Hatch** for environment and dependency management.
+
+Do not run `python` or `pip` directly.
+
+Before running code on this project, you must be able to run `hatch --version` and get a correct output.
+
+If not, ask the user where Hatch is or if they want to install it. For installation instructions, refer to https://hatch.pypa.io/latest/install/#installation.
+
+### Run scripts with test dependencies
+
+hatch -e test run python SCRIPT.py
+
+### Open a shell with test dependencies
+
+hatch -e test shell
+
+### Install temporary dependencies (for experiments only)
+
+uv pip install PACKAGE
+
+### Delete the environment
+
+hatch env prune
+
+## Tests
+
+Tests run via Hatch and support pytest arguments.
+
+Prefer running tests on a specific module or using `-k`, since the full suite is large.
+
+### Run unit tests
+
+hatch run test:unit
+
+### Run integration tests
+
+hatch run test:integration
+
+## Quality Checks
+
+### Type checking with mypy
+hatch run test:types
+
+To fix type issues, avoid `type: ignore`, casts, or assertions when possible. If they are necessary, explain why.
+
+### Format and lint
+hatch run fmt
+
+## Release Notes
+
+Every user-facing PR (not docs, not CI) must include a release note:
+
+hatch run release-note SHORT_DESCRIPTION
+
+Edit the generated file in `releasenotes/notes/`. Release notes use reStructuredText formatting; see the [release notes section in CONTRIBUTING.md](CONTRIBUTING.md#release-notes) for details.

CITATION.cff

@@ -0,0 +1,30 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it using these metadata."
+title: "Haystack: the end-to-end NLP framework for pragmatic builders"
+date-released: 2019-11-14
+url: "https://github.com/deepset-ai/haystack"
+authors:
+- family-names: Pietsch
+  given-names: Malte
+- family-names: Möller
+  given-names: Timo
+- family-names: Kostic
+  given-names: Bogdan
+- family-names: Risch
+  given-names: Julian
+- family-names: Pippi
+  given-names: Massimiliano
+- family-names: Jobanputra
+  given-names: Mayank
+- family-names: Zanzottera
+  given-names: Sara
+- family-names: Cerza
+  given-names: Silvano
+- family-names: Blagojevic
+  given-names: Vladimir
+- family-names: Stadelmann
+  given-names: Thomas
+- family-names: Soni
+  given-names: Tanay
+- family-names: Lee
+  given-names: Sebastian

CLAUDE.md

@@ -0,0 +1,3 @@
+# CLAUDE.md
+
+Before you start working on this repository, read the AGENTS.md file and follow all the instructions.

CONTRIBUTING.md

@@ -0,0 +1,463 @@
+# Contributing to Haystack
+
+First off, thanks for taking the time to contribute! :blue_heart:
+
+All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents)
+for different ways to help and details about how this project handles them. Please make sure to read
+the relevant section before making your contribution. It will make it a lot easier for us maintainers
+and smooth out the experience for all involved. The community looks forward to your contributions!
+
+> [!TIP]
+> If you like Haystack but just don't have time to contribute, that's fine. There are other easy ways to support the
+> project and show your appreciation: star this repository ⭐, mention Haystack at local meetups and tell your
+> friends/colleagues, or share what you build and tag [Haystack on X (Twitter)](https://x.com/Haystack_ai) and
+> [Haystack on LinkedIn](https://www.linkedin.com/showcase/haystack-ai-framework) — we'd love to see it!
+
+## Your first PR — high-level to-do list
+
+Use this checklist to stay on track for your first code PR:
+
+- **Pick an issue** — Choose one labeled [good first issue](https://github.com/deepset-ai/haystack/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or [contributions wanted](https://github.com/deepset-ai/haystack/issues?q=is%3Aissue%20state%3Aopen%20label%3A"Contributions%20wanted!"). Avoid issues marked or commented as [handled internally](#issues-not-open-for-external-contributions).
+- **Fork and clone** — [Clone the repository](#clone-the-git-repository), run `pre-commit install`, and create a branch.
+- **Set up and run** — [Set up your development environment](#setting-up-your-development-environment), run unit tests with `hatch run test:unit` and run quality checks with `hatch run test:types` and `hatch run fmt`.
+- **Implement and test** — Make your changes, add or update tests as needed, and ensure tests and pre-commit checks pass locally.
+- **Documentation** — If your change adds or alters user-facing behavior, add a new docs page or update the relevant one in `docs-website/` (edit under `docs/` for the next release; add new pages to `sidebars.js`). See the [Documentation Contributing Guide](docs-website/CONTRIBUTING.md) for where to edit, frontmatter, and navigation.
+- **Release notes** — Add a release note under `releasenotes/notes` with `hatch run release-note your-change-name` (see [Release notes](#release-notes)); maintainers can add `ignore-for-release-notes` for tests-only or CI-only changes.
+- **Open the PR** — Use a [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) title, fill the [PR template](.github/pull_request_template.md), and if the PR was fully AI-generated, add a [short disclaimer](#using-ai-assistants-to-contribute). Enable "Allow edits and access to secrets by maintainers" on the PR.
+- **Sign the CLA** — A [Contributor Licence Agreement (CLA)](https://cla-assistant.io/deepset-ai/haystack) is required for all contributions. Sign when prompted so your PR is ready for review (see [CLA](#contributor-licence-agreement-cla)).
+- **Once the PR is open** — Fix any [CI](#ci-continuous-integration) failures and address review feedback.
+
+**Table of Contents**
+
+- [Contributing to Haystack](#contributing-to-haystack)
+  - [Your first PR — high-level to-do list](#your-first-pr--high-level-to-do-list)
+  - [Code of Conduct](#code-of-conduct)
+  - [I Have a Question](#i-have-a-question)
+  - [Reporting Bugs](#reporting-bugs)
+    - [Before Submitting a Bug Report](#before-submitting-a-bug-report)
+    - [How Do I Submit a Good Bug Report?](#how-do-i-submit-a-good-bug-report)
+  - [Suggesting Enhancements](#suggesting-enhancements)
+    - [Before Submitting an Enhancement](#before-submitting-an-enhancement)
+    - [How Do I Submit a Good Enhancement Suggestion?](#how-do-i-submit-a-good-enhancement-suggestion)
+  - [Contributing to Documentation](#contributing-to-documentation)
+  - [Contribute code](#contribute-code)
+    - [Where to start](#where-to-start)
+    - [Issues not open for external contributions](#issues-not-open-for-external-contributions)
+    - [Example high-quality contributions](#example-high-quality-contributions)
+    - [Using AI assistants to contribute](#using-ai-assistants-to-contribute)
+    - [Setting up your development environment](#setting-up-your-development-environment)
+    - [Clone the git repository](#clone-the-git-repository)
+    - [Run the tests locally](#run-the-tests-locally)
+  - [Requirements for Pull Requests](#requirements-for-pull-requests)
+    - [Release notes](#release-notes)
+  - [CI (Continuous Integration)](#ci-continuous-integration)
+  - [Working from GitHub forks](#working-from-github-forks)
+  - [Writing tests](#writing-tests)
+    - [Unit test](#unit-test)
+    - [Integration test](#integration-test)
+    - [End to End (e2e) test](#end-to-end-e2e-test)
+    - [Slow/unstable integration tests (for maintainers)](#slowunstable-integration-tests-for-maintainers)
+  - [Contributor Licence Agreement (CLA)](#contributor-licence-agreement-cla)
+
+## Code of Conduct
+
+This project and everyone participating in it is governed by our [Code of Conduct](code_of_conduct.txt).
+By participating, you are expected to uphold this code. Please report unacceptable behavior to haystack@deepset.ai.
+
+## I Have a Question
+
+Before you ask a question, it is best to search for existing [Issues](https://github.com/deepset-ai/haystack/issues) that might help you. In case you have
+found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to
+search the internet for answers first.
+
+If you then still feel the need to ask a question and need clarification, you can use [Haystack's Discord Server](https://discord.com/invite/xYvH6drSmA).
+
+## Reporting Bugs
+
+### Before Submitting a Bug Report
+
+A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to
+investigate carefully, collect information, and describe the issue in detail in your report. Please complete the
+following steps in advance to help us fix any potential bugs as fast as possible.
+
+- Make sure that you are using the latest version.
+- Determine if your bug is really a bug and not an error on your side, for example, using incompatible versions.
+  Make sure that you have read the [documentation](https://docs.haystack.deepset.ai/docs/intro). If you are looking
+  for support, you might want to check [this section](#i-have-a-question).
+- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there
+  is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/deepset-ai/haystack/issues).
+- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have
+  discussed the issue.
+- Collect information about the bug:
+  - OS, Platform and Version (Windows, Linux, macOS, x86, ARM)
+  - Version of Haystack and the integrations you're using
+  - Possibly your input and the output
+  - If you can reliably reproduce the issue, a snippet of code we can use
+
+### How Do I Submit a Good Bug Report?
+
+> [!IMPORTANT]
+> You must never report security-related issues, vulnerabilities, or bugs, including sensitive information, to the issue tracker, or elsewhere in public. Instead, sensitive bugs must be reported using [this link](https://github.com/deepset-ai/haystack/security/advisories/new).
+
+We use GitHub issues to track bugs and errors. If you run into an issue with the project:
+
+- Open an [Issue of type Bug Report](https://github.com/deepset-ai/haystack/issues/new?assignees=&labels=bug&projects=&template=bug_report.md&title=).
+- Explain the behavior you would expect and the actual behavior.
+- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to
+  recreate the issue on their own. This usually includes your code. For good bug reports, you should isolate the problem
+  and create a reduced test case.
+- Provide the information you collected in the previous section.
+
+Once it's filed:
+
+- The project team will label the issue accordingly.
+- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no
+  obvious way to reproduce the issue, the team will ask you for those steps.
+- If the team is able to reproduce it, the issue will be scheduled for a fix or left to be
+  [picked up by a community contributor](https://github.com/deepset-ai/haystack/issues?q=is%3Aissue%20state%3Aopen%20label%3A"Contributions%20wanted!").
+
+## Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion, including new integrations and improvements
+to existing ones. Following these guidelines will help maintainers and the community to understand your suggestion and
+find related suggestions.
+
+### Before Submitting an Enhancement
+
+- Make sure that you are using the latest version.
+- Read the [documentation](https://docs.haystack.deepset.ai/docs/intro) carefully and find out if the functionality
+  is already covered, possibly via particular configuration parameters.
+- Perform a [search](https://github.com/deepset-ai/haystack/issues) to see if the enhancement has already been suggested. If it has, add a comment to the
+  existing issue instead of opening a new one.
+- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to
+  convince the project's developers of the merits of this feature. Keep in mind that we want features that will be
+  useful to the majority of our users and not just a small subset. If you're just targeting a minority of users,
+  consider writing and distributing the integration on your own.
+
+### How Do I Submit a Good Enhancement Suggestion?
+
+Enhancement suggestions are tracked as GitHub issues of type [Feature request](https://github.com/deepset-ai/haystack/issues/new?template=feature_request.md).
+
+- Use a **clear and descriptive title** for the issue to identify the suggestion.
+- Fill in the issue following the template
+
+## Contributing to Documentation
+
+If you'd like to improve the documentation by fixing errors, clarifying explanations, adding examples, or creating new guides, see the [Documentation Contributing Guide](docs-website/CONTRIBUTING.md).
+
+## Contribute code
+
+> [!IMPORTANT]
+> When contributing to this project, you must agree that you have authored or carefully reviewed 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+
+### Where to start
+
+If this is your first code contribution, a good starting point is looking for an open issue that's marked with the label
+["good first issue"](https://github.com/deepset-ai/haystack/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22).
+The core contributors periodically mark certain issues as good for first-time contributors. Those issues are usually
+limited in scope, easily fixable and low priority, so there is absolutely no reason why you should not try fixing them.
+It's a good excuse to start looking into the project and a safe space to experiment and fail: if you don't get the
+grasp of something, pick another one! Once you become comfortable contributing to Haystack, you can have a look at the
+list of issues marked as [contributions wanted](https://github.com/orgs/deepset-ai/projects/14/views/1) to look for your
+next contribution!
+
+### Issues not open for external contributions
+
+Some issues are handled internally by the core team and are **not open for external contributions**. You may see a
+comment on such issues like:
+
+> 👋 Hello there! This issue will be handled internally and isn't open for external contributions. If you'd like to contribute, please take a look at issues labeled **contributions welcome** or **good first issue**. We'd really appreciate it!
+
+> [!WARNING]
+> **Please do not open pull requests for issues that are marked or commented as handled internally.** Your work may not be merged. Instead, look for issues labeled [good first issue](https://github.com/deepset-ai/haystack/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or [contributions wanted](https://github.com/deepset-ai/haystack/issues?q=is%3Aissue%20state%3Aopen%20label%3A"Contributions%20wanted!") — we'd love your help there!
+
+### Example high-quality contributions
+
+Looking at strong pull requests is a great way to learn our standards. Example high-quality PRs: [#9270](https://github.com/deepset-ai/haystack/pull/9270), [#9227](https://github.com/deepset-ai/haystack/pull/9227), [#9271](https://github.com/deepset-ai/haystack/pull/9271), [#8648](https://github.com/deepset-ai/haystack/pull/8648), [#8767](https://github.com/deepset-ai/haystack/pull/8767). Use them as references for structure, testing, documentation, and how to describe changes in the PR description and release notes.
+
+### Using AI assistants to contribute
+
+You may use AI assistants or agents to help you implement a contribution. Please use them wisely:
+
+- **Review and understand** all generated code before submitting. You are responsible for the contribution.
+- **Run tests and checks** locally (e.g. `hatch run test:unit`, `hatch run fmt`) so your PR meets our quality bar.
+- **If your PR was fully AI-generated**, add a short disclaimer in the PR description, for example: *"This PR was
+  fully generated with an AI assistant. I have reviewed the changes and run the relevant tests."*
+
+This helps maintainers and keeps the project ready for both human and AI contributors.
+
+### Setting up your development environment
+
+*To run Haystack tests locally, ensure your development environment uses Python >=3.10 and <3.14.*
+
+Haystack makes heavy use of [Hatch](https://hatch.pypa.io/latest/), a Python project manager that we use to set up the
+virtual environments, build the project, and publish packages. As you can imagine, the first step towards becoming a
+Haystack contributor is installing Hatch. There are a variety of installation methods depending on your operating system
+platform, version, and personal taste: please have a look at [this page](https://hatch.pypa.io/latest/install/#installation)
+and keep reading once you can run from your terminal:
+
+```console
+$ hatch --version
+Hatch, version 1.14.1
+```
+
+You create a new virtual environment for Haystack with `hatch` by running:
+
+```console
+$ hatch shell
+```
+
+### Clone the git repository
+
+You won't be able to make changes directly to this repo, so the first step is to [create a fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo).
+Once your fork is ready, you can clone a local copy with:
+
+```console
+$ git clone https://github.com/YOUR-USERNAME/haystack
+```
+
+If everything worked, you should be able to do something like this (the output might be different):
+
+```console
+$ cd haystack
+
+$ hatch version
+2.3.0-rc0
+```
+
+Last, enter the virtual environment:
+
+```console
+$ hatch shell
+```
+
+and install the pre-commit hooks:
+
+```console
+pre-commit install
+```
+
+Note: It is important to run `pre-commit install` inside the virtual environment created with `hatch shell`. If you don't, you'll get an error message like this: `pre-commit: command not found`.
+
+pre-commit will run some tasks right before all `git commit` operations. From now on, your `git commit` output for Haystack should look something like this:
+
+```console
+> git commit -m "test"
+check python ast.........................................................Passed
+check json...........................................(no files to check)Skipped
+check for merge conflicts................................................Passed
+check that scripts with shebangs are executable..........................Passed
+check toml...........................................(no files to check)Skipped
+check yaml...........................................(no files to check)Skipped
+fix end of files.........................................................Passed
+mixed line ending........................................................Passed
+don't commit to branch...................................................Passed
+trim trailing whitespace.................................................Passed
+ruff.....................................................................Passed
+codespell................................................................Passed
+Lint GitHub Actions workflow files...................(no files to check)Skipped
+[massi/contrib d18a2577] test
+ 2 files changed, 178 insertions(+), 45 deletions(-)
+```
+
+### Run the tests locally
+
+Tests will automatically run in our CI for every commit you push to your PR on Github. In order to save precious CI time, we encourage you to run the tests locally before pushing new commits to Github. From the root of the git repository, you can run all the unit tests like this:
+
+```sh
+hatch run test:unit
+```
+
+Hatch will create a dedicated virtual environment, sync the required dependencies and run all the unit tests from the
+project. If you want to run a subset of the tests or even one test in particular, `hatch` will accept all the
+options you would normally pass to `pytest`, for example:
+
+```sh
+# run one test method from a specific test class in a test file
+hatch run test:unit test/test_logging.py::TestSkipLoggingConfiguration::test_skip_logging_configuration
+```
+
+### Run code quality checks locally
+
+We also use tools to ensure consistent code style, quality, and static type checking. The quality of your code will be
+tested by the CI, but once again, running the checks locally will speed up the review cycle.
+
+
+To check for static type errors, run:
+```sh
+hatch run test:types
+```
+
+To format your code and perform linting using Ruff (with automatic fixes), run:
+```sh
+hatch run fmt
+```
+
+
+## Requirements for Pull Requests
+
+To ease the review process, please follow the instructions in this paragraph when creating a Pull Request:
+
+- For the title, use the [conventional commit convention](https://www.conventionalcommits.org/en/v1.0.0/).
+- For the body, follow the existing [pull request template](https://github.com/deepset-ai/haystack/blob/main/.github/pull_request_template.md) to describe and document your changes.
+- If you used an AI assistant and the PR was **fully AI-generated**, include a brief disclaimer in the PR description
+  (see [Using AI assistants to contribute](#using-ai-assistants-to-contribute)).
+
+### Release notes
+
+Each PR must include a release notes file under the `releasenotes/notes` path created with `reno`, and a CI check will
+fail if that's not the case. Pull requests with changes limited to tests, code comments or docstrings, and changes to
+the CI/CD systems can be labeled with `ignore-for-release-notes` by a maintainer in order to bypass the CI check.
+
+For example, if your PR is bumping the `transformers` version in the `pyproject.toml` file, that's something that
+requires release notes. To create the corresponding file, from the root of the repo run:
+
+```
+$ hatch run release-note bump-transformers-to-4-31
+```
+
+A release notes file in YAML format will be created in the appropriate folder, appending a unique id to the name of the
+release note you provided (in this case, `bump-transformers-to-4-31`). To add the actual content of the release notes,
+you must edit the file that's just been created. In the file, you will find multiple sections along with an explanation
+of what they're for. You have to remove all the sections that don't fit your release notes, in this case for example
+you would fill in the `enhancements` section to describe the change:
+
+```yaml
+enhancements:
+  - |
+    Upgrade transformers to the latest version 4.31.0 so that Haystack can support the new LLama2 models.
+```
+
+Each section of the YAML file must follow [reStructuredText formatting](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html).
+
+For inline code, use double backticks to wrap the code.
+```
+``OpenAIChatGenerator``
+```
+
+For code blocks, use the [code block directive](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-code-block).
+
+```
+.. code:: python
+  from haystack.dataclasses import ChatMessage
+
+  message = ChatMessage.from_user("Hello!")
+  print(message.text)
+```
+
+You can now add the file to the same branch containing the code changes. Your release note will be part of your pull
+request and reviewed along with any code you changed.
+
+## CI (Continuous Integration)
+
+We use GitHub Action for our Continuous Integration tasks. This means that as soon as you open a PR, GitHub will start
+executing some workflows on your changes, like automated tests, linting, formatting, api docs generation, etc.
+
+If all goes well, at the bottom of your PR page you should see something like this, where all checks are green.
+
+![Successful CI](images/ci-success.png)
+
+If you see some red checks (like the following), then something didn't work, and action is needed on your side.
+
+![Failed CI](images/ci-failure-example.png)
+
+Click on the failing test and see if there are instructions at the end of the logs of the failed test.
+For example, in the case above, the CI will give you instructions on how to fix the issue.
+
+![Logs of failed CI, with instructions for fixing the failure](images/ci-failure-example-instructions.png)
+
+## Working from GitHub forks
+
+To help maintainers, we usually ask contributors to grant us push access to their fork.
+
+To do so, please verify that "Allow edits and access to secrets by maintainers" on the PR preview page is checked
+(you can check it later on the PR's sidebar once it's created).
+
+![Allow access to your branch to maintainers](images/first_time_contributor_enable_access.png)
+
+## Writing tests
+
+We formally define three scopes for tests in Haystack with different requirements and purposes:
+
+### Unit test
+- Tests a single logical concept
+- Execution time is a few milliseconds
+- Any external resource is mocked
+- Always returns the same result
+- Can run in any order
+- Runs at every commit in PRs, automated through `hatch run test:unit`
+- Can run locally with no additional setup
+- **Goal: being confident in merging code**
+
+### Integration test
+- Tests a single logical concept
+- Execution time is a few seconds
+- It uses external resources that must be available before execution
+- When using models, cannot use inference
+- Always returns the same result or an error
+- Can run in any order
+- Runs at every commit in PRs, automated through `hatch run test:integration`
+- Can run locally with some additional setup (e.g. Docker)
+- **Goal: being confident in merging code**
+
+### End to End (e2e) test
+- Tests a sequence of multiple logical concepts
+- Execution time has no limits (can be always on)
+- Can use inference
+- Evaluates the results of the execution or the status of the system
+- It uses external resources that must be available before execution
+- Can return different results
+- Can be dependent on the order
+- Can be wrapped into any process execution
+- Runs outside the development cycle (nightly or on demand)
+- Might not be possible to run locally due to system and hardware requirements
+- **Goal: being confident in releasing Haystack**
+
+### Slow/unstable Integration Tests (for maintainers)
+
+To keep the CI stable and reasonably fast, we run certain tests in a separate workflow.
+
+We use `@pytest.mark.slow` for tests that clearly meet one or more of the following conditions:
+- Unstable (such as call unstable external services)
+- Slow (such as model inference on CPU)
+- Require special setup (such as installing system dependencies, running Docker containers).
+
+⚠️ The main goal of this separation is to keep the regular integration tests fast and **stable**.
+
+We should try to avoid including too many modules in the Slow Integration Tests workflow: doing so may reduce its effectiveness.
+
+#### How does it work?
+
+These tests are executed by the [Slow Integration Tests workflow](.github/workflows/slow.yml).
+
+The workflow always runs, but the tests only execute when:
+
+- There are changes to relevant files (as listed in the [workflow file](.github/workflows/slow.yml)).
+  **Important**: If you mark a test but do not include both the test file and the file to be tested in the list, the test won't run automatically.
+- The workflow is scheduled (runs nightly).
+- The workflow is triggered manually (with the "Run workflow" button on [this page](https://github.com/deepset-ai/haystack/actions/workflows/slow.yml)).
+- The PR has the "run-slow-tests" label (you can use this label to trigger the tests even if no relevant files are changed).
+- The push is to a release branch.
+
+If none of the above conditions are met, the workflow completes successfully without running tests to satisfy Branch Protection rules.
+
+*Hatch commands for running Integration Tests*:
+- `hatch run test:integration` runs all integrations tests (fast + slow).
+- `hatch run test:integration-only-fast` skips the slow tests.
+- `hatch run test:integration-only-slow` runs only slow tests.
+
+## Contributor Licence Agreement (CLA)
+
+Significant contributions to Haystack require a Contributor License Agreement (CLA). If the contribution requires a CLA,
+we will get in contact with you. CLAs are quite common among company-backed open-source frameworks, and our CLA’s wording
+is similar to other popular projects, like [Rasa](https://cla-assistant.io/RasaHQ/rasa) or
+[Google's Tensorflow](https://cla.developers.google.com/clas/new?domain=DOMAIN_GOOGLE&kind=KIND_INDIVIDUAL)
+(retrieved 4th November 2021).
+
+The agreement's main purpose is to protect the continued open use of Haystack. At the same time, it also helps in
+\protecting you as a contributor. Contributions under this agreement will ensure that your code will continue to be
+open to everyone in the future (“You hereby grant to Deepset **and anyone** [...]”) as well as remove liabilities on
+your end (“you provide your Contributions on an AS IS basis, without warranties or conditions of any kind [...]”). You
+can find the Contributor Licence Agreement [here](https://cla-assistant.io/deepset-ai/haystack).
+
+If you have further questions about the licensing, feel free to reach out to contributors@deepset.ai.

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2021 deepset GmbH
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,114 @@
+<div align="center">
+  <a href="https://haystack.deepset.ai/"><img src="https://raw.githubusercontent.com/deepset-ai/haystack/main/images/banner.png" alt="Blue banner with the Haystack logo and the text ‘haystack by deepset – The Open Source AI Framework for Production Ready RAG & Agents’ surrounded by abstract icons representing search, documents, agents, pipelines, and cloud systems."></a>
+
+|         |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| CI/CD   | [![Tests](https://github.com/deepset-ai/haystack/actions/workflows/tests.yml/badge.svg)](https://github.com/deepset-ai/haystack/actions/workflows/tests.yml) [![types - Mypy](https://img.shields.io/badge/types-Mypy-blue.svg)](https://github.com/python/mypy) [![Coverage badge](https://raw.githubusercontent.com/deepset-ai/haystack/python-coverage-comment-action-data/badge.svg)](https://htmlpreview.github.io/?https://github.com/deepset-ai/haystack/blob/python-coverage-comment-action-data/htmlcov/index.html) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) |
+| Docs    | [![Website](https://img.shields.io/website?label=documentation&up_message=online&url=https%3A%2F%2Fdocs.haystack.deepset.ai)](https://docs.haystack.deepset.ai)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| Package | [![PyPI](https://img.shields.io/pypi/v/haystack-ai)](https://pypi.org/project/haystack-ai/) ![PyPI - Downloads](https://img.shields.io/pypi/dm/haystack-ai?color=blue&logo=pypi&logoColor=gold) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/haystack-ai?logo=python&logoColor=gold) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/haystack-ai.svg)](https://anaconda.org/conda-forge/haystack-ai) [![GitHub](https://img.shields.io/github/license/deepset-ai/haystack?color=blue)](LICENSE) [![License Compliance](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml/badge.svg)](https://github.com/deepset-ai/haystack/actions/workflows/license_compliance.yml) [![HVTrust](https://hvtracker.net/badge/haystack.svg)](https://hvtracker.net/agents/haystack/) [![Evidence Grade](https://hvtracker.net/badge/haystack-grade.svg)](https://hvtracker.net/agents/haystack/) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/13067/badge)](https://www.bestpractices.dev/projects/13067)|
+| Meta    | [![Discord](https://img.shields.io/discord/993534733298450452?logo=discord)](https://discord.com/invite/qZxjM4bAHU) [![Twitter Follow](https://img.shields.io/twitter/follow/haystack_ai)](https://twitter.com/haystack_ai)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+</div>
+
+[Haystack](https://haystack.deepset.ai/) is an open-source AI orchestration framework for building production-ready LLM applications in Python.
+
+Design modular pipelines and agent workflows with explicit control over retrieval, routing, memory, and generation. Build scalable RAG systems, multimodal applications, semantic search, question answering, and autonomous agents, all in a transparent architecture that lets you experiment, customize deeply, and deploy with confidence.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Documentation](#documentation)
+- [Features](#features)
+- [Haystack Enterprise: Support & Platform](#haystack-enterprise-support--platform)
+- [Telemetry](#telemetry)
+- [🖖 Community](#-community)
+- [Contributing to Haystack](#contributing-to-haystack)
+- [Organizations using Haystack](#organizations-using-haystack)
+
+
+## Installation
+
+The simplest way to get Haystack is via pip:
+
+```sh
+pip install haystack-ai
+```
+
+Install nightly pre-releases to try the newest features:
+```sh
+pip install --pre haystack-ai
+```
+
+Haystack supports multiple installation methods, including Docker images. For a comprehensive guide, please refer
+to the [documentation](https://docs.haystack.deepset.ai/docs/installation).
+
+## Documentation
+
+If you're new to the project, check out ["What is Haystack?"](https://haystack.deepset.ai/overview/intro) then go
+through the ["Get Started Guide"](https://haystack.deepset.ai/overview/quick-start) and build your first LLM application
+in a matter of minutes. Keep learning with the [tutorials](https://haystack.deepset.ai/tutorials). For more advanced
+use cases, or just to get some inspiration, you can browse our Haystack recipes in the
+[Cookbook](https://haystack.deepset.ai/cookbook).
+
+At any given point, hit the [documentation](https://docs.haystack.deepset.ai/docs/intro) to learn more about Haystack, what it can do for you, and the technology behind.
+
+## Features
+
+**Built for context engineering**  
+Design flexible systems with explicit control over how information is retrieved, ranked, filtered, combined, structured, and routed before it reaches the model. Define pipelines and agent workflows where retrieval, memory, tools, and generation are transparent and traceable.
+
+**Model- and vendor-agnostic**  
+Integrate with OpenAI, Mistral, Anthropic, Cohere, Hugging Face, Azure OpenAI, AWS Bedrock, local models, and many others. Swap models or infrastructure components without rewriting your system.
+
+**Modular and customizable**  
+Use built-in components for retrieval, indexing, tool calling, memory, and evaluation, or create your own. Add loops, branches, and conditional logic to precisely control how context moves through your pipelines and agent workflows.
+
+**Extensible ecosystem**  
+Build and share custom components through a consistent interface that makes it easy for the community and third parties to extend Haystack and contribute to an open ecosystem.
+
+> [!TIP]
+>
+> Would you like to deploy and serve Haystack pipelines as **REST APIs** or **MCP servers**? [Hayhooks](https://github.com/deepset-ai/hayhooks) provides a simple way for you to wrap pipelines and agents with custom logic and expose them through HTTP endpoints or MCP. It also supports OpenAI-compatible chat completion endpoints and works with chat UIs like [open-webui](https://openwebui.com/).
+
+## Haystack Enterprise: Support & Platform
+
+Get expert support from the Haystack team, build faster with enterprise-grade templates, and scale securely with deployment guides for cloud and on-prem environments with **Haystack Enterprise Starter**. Read more about it in the [announcement post](https://haystack.deepset.ai/blog/announcing-haystack-enterprise).
+
+👉 [Get Haystack Enterprise Starter](https://www.deepset.ai/products-and-services/haystack-enterprise-starter?utm_source=github.com&utm_medium=referral&utm_campaign=haystack_enterprise)
+
+Need a managed production setup for Haystack? The **Haystack Enterprise Platform** helps you build, test, deploy and operate Haystack pipelines with built-in observability, collaboration, governance, and access controls. It’s available as a managed cloud service or as a self-hosted solution.
+
+👉 Learn more about [Haystack Enterprise Platform](https://www.deepset.ai/products-and-services/haystack-enterprise-platform?utm_campaign=developer-relations&utm_source=haystack&utm_medium=readme) or [try it free](https://www.deepset.ai/haystack-enterprise-platform-trial?utm_campaign=developer-relations&utm_source=haystack&utm_medium=readme)
+
+## Telemetry
+
+Haystack collects **anonymous** usage statistics of pipeline components. We receive an event every time these components are initialized. This way, we know which components are most relevant to our community.
+
+Read more about telemetry in Haystack or how you can opt out in [Haystack docs](https://docs.haystack.deepset.ai/docs/telemetry).
+
+## 🖖 Community
+
+If you have a feature request or a bug report, feel free to open an [issue in GitHub](https://github.com/deepset-ai/haystack/issues). We regularly check these, so you can expect a quick response. If you'd like to discuss a topic or get more general advice on how to make Haystack work for your project, you can start a thread in [Github Discussions](https://github.com/deepset-ai/haystack/discussions) or our [Discord channel](https://discord.com/invite/VBpFzsgRVF). We also check [𝕏 (Twitter)](https://twitter.com/haystack_ai) and [Stack Overflow](https://stackoverflow.com/questions/tagged/haystack).
+
+## Contributing to Haystack
+
+We are very open to the community's contributions - be it a quick fix of a typo, or a completely new feature! You don't need to be a Haystack expert to provide meaningful improvements. To learn how to get started, check out our [Contributor Guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md) first.
+
+There are several ways you can contribute to Haystack:
+- Contribute to the main Haystack project
+- Contribute an integration on [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations)
+- Contribute to the documentation in [haystack/docs-website](https://github.com/deepset-ai/haystack/tree/main/docs-website)
+
+> [!TIP]
+>👉 **[Check out the full list of issues that are open to contributions](https://github.com/orgs/deepset-ai/projects/14)**
+
+## Organizations using Haystack
+
+Haystack is used by thousands of teams building production AI systems across industries, including:
+
+- **Technology & AI Infrastructure**: [Apple](https://www.apple.com/), [Meta](https://www.meta.com/about), [Databricks](https://www.databricks.com/), [NVIDIA](https://developer.nvidia.com/blog/reducing-development-time-for-intelligent-virtual-assistants-in-contact-centers/), [Intel](https://github.com/intel/open-domain-question-and-answer#readme)
+- **Public Sector AI Initiatives**: [European Commission](https://commission.europa.eu/index_en), [German Federal Ministry of Research, Technology, and Space (BMFTR)](https://www.deepset.ai/case-studies/german-federal-ministry-research-technology-space-bmftr), [PD, Baden-Württemberg State](https://www.pd-g.de/)
+- **Enterprise & Industrial AI Applications**: [Airbus](https://www.deepset.ai/case-studies/airbus), [Lufthansa Industry Solutions](https://haystack.deepset.ai/blog/lufthansa-user-story), [Infineon](https://www.infineon.com/), [LEGO](https://github.com/larsbaunwall/bricky#readme), [Comcast](https://arxiv.org/html/2405.00801v2), [Accenture](https://www.accenture.com/), [TELUS Agriculture & Consumer Goods](https://www.telus.com/agcg/en)
+- **Knowledge & Content Platforms**: [Netflix](https://netflix.com), [ZEIT Online](https://www.deepset.ai/case-studies/zeit-online), [Rakuten](https://www.rakuten.com/), [Oxford University Press](https://corp.oup.com/), [Manz](https://www.deepset.ai/case-studies/manz), [YPulse](https://www.deepset.ai/case-studies/ypulse)
+
+
+Are you also using Haystack? Open a PR or [tell us your story](https://forms.gle/Mm3G1aEST3GAH2rn8)

README.wehub.md

@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目：`deepset-ai/haystack`
+- 原始仓库：https://github.com/deepset-ai/haystack
+- 导入方式：上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源，不代表 WeHub 是原项目作者

SECURITY.md

@@ -0,0 +1,67 @@
+# Security Policy
+
+## Report a Vulnerability
+
+If you have found a security vulnerability in Haystack, please report via email to
+[opensource-security@deepset.ai](mailto:opensource-security@deepset.ai).
+
+In your message, please include:
+
+1. Reproducible steps to trigger the vulnerability.
+2. An explanation of what makes you think there is a vulnerability.
+3. Any information you may have on active exploitations of the vulnerability (zero-day).
+4. An explanation of why you believe the vulnerability is not out of scope. See the Out of Scope section below.
+
+We encourage reports that are meaningful, high-impact, and reviewed by a human before submission. Fully automated or AI-generated reports submitted without human review and validation are unlikely to meet this bar and risk being declined.
+
+## Out of Scope
+
+Haystack is a framework intended to run inside a trusted execution environment. It assumes that the application built with it has already validated and sanitized user-supplied input before passing it to the framework. Validation and sanitization of input, for example URLs, file paths, filter expressions, and queries, are the responsibility of the application, not Haystack.
+
+Any vulnerability that can only be triggered by passing unsanitized, attacker-controlled input to Haystack is considered out of scope. This reflects a conscious design decision after evaluating the trade-offs and risks: as a framework, Haystack cannot and should not enforce input validation on behalf of every application that uses it.
+
+The following areas have been deliberately scoped out below and we ask that you read them carefully before submitting.
+
+### Pipeline Serialization
+
+Haystack pipelines in YAML are a serialization format for executable pipelines. `Pipeline.loads()`, `deserialize_callable()`, and `import_class_by_name()` are designed to load user-defined components, filters, and callables. They intentionally support dynamic imports at runtime to enable extensibility.
+
+**Loading a pipeline from an untrusted source is unsafe by design.** This is not a hidden weakness but the expected consequence of a system that lets users bring their own code. The security responsibility lies with the operator: pipeline definitions must be treated as code, stored and transmitted with the same controls applied to source code, and never loaded from untrusted or user-controlled input without review.
+Reports that demonstrate, for example, `import_class_by_name()` can import arbitrary modules given a pipeline that an operator chose to load are out of scope.
+
+The same trust boundary applies to serialized snapshots, including `PipelineSnapshot` and `AgentSnapshot`. A snapshot is operator-controlled state that is intentionally human-readable and editable (for example, to inspect or modify a paused execution before resuming it). Resuming from a snapshot deserializes its contents without integrity verification, by design. Snapshots must therefore be stored and access-controlled as trusted artifacts, exactly as pipeline definitions are. Reports that rely on modifying a snapshot an operator chose to resume from are out of scope.
+
+However, if you find a way to achieve arbitrary code execution that does *not* rely on an operator loading an untrusted pipeline (for example, a memory-safety issue in the parser itself, or a class of pipelines that triggers unintended behaviour in the Haystack runtime), that finding is in scope.
+
+### SSRF via URL-fetching Components (e.g. `LinkContentFetcher`)
+
+Components such as `LinkContentFetcher` accept URLs and fetch their content. They do not validate or restrict which hosts or IP ranges are reachable. This is intentional: restricting network access at the framework layer would break legitimate use cases and would provide only a weak, bypassable control anyway. Just as developers would not expect the [requests](https://github.com/psf/requests) library to enforce SSRF protections, Haystack components that perform HTTP requests do not gate which hosts are reachable by default.
+
+**SSRF prevention is the responsibility of the application and the network layer.** The expected mitigations are: resolving the requested hostname before fetching and rejecting requests whose resolved IP falls within private or reserved CIDR ranges, enforcing egress firewall rules at the infrastructure level, and running Haystack in a network-isolated environment where internal services are not reachable from the component's runtime context.
+
+We are aware that demonstrating an open fetch to `http://169.254.169.254` or similar metadata endpoints is straightforward. Reports that do so without the application-layer controls described above in place are out of scope. If you find a bypass of a network control that *is* in place, or a vulnerability in how Haystack processes the fetched content (for example, a parsing issue that leads to unintended behaviour), that finding is in scope.
+
+### Prompt Injection via Documents or Metadata
+
+Haystack passes documents, metadata, and other retrieved content into prompt templates at runtime. If an attacker can store prompt-injection payloads in a document store or retrieval source that a Haystack pipeline later reads and inserts into a prompt, the LLM may follow those instructions.
+
+**Prompt injection detection and mitigation are the responsibility of the application.** Haystack provides building blocks, including classifier components and documented patterns, for application developers to implement prompt injection defences. Without such defences in place, retrieved content can reach the LLM unfiltered.
+
+We are aware that crafting a document containing `Ignore previous instructions...` and showing it surfaces in a generated response is straightforward to demonstrate. Reports of that form are out of scope. If you find a vulnerability in a Haystack component that *bypasses* a defence a developer has wired into their pipeline, or causes unintended behaviour outside of the LLM interaction (for example, exfiltrating data through Haystack's own APIs), that finding may be in scope.
+
+---
+
+If you are uncertain whether a finding falls within scope, feel free to reach out before submitting a full report.
+
+## Vulnerability Response
+
+We aim to review your report within 5 business days where we do a preliminary analysis
+to confirm that the vulnerability is plausible. Otherwise, we'll decline the report.
+
+We won't disclose any information you share with us but we'll use it to get the issue
+fixed or to coordinate a vendor response, as needed.
+
+We'll keep you updated of the status of the issue.
+
+Our goal is to disclose bugs as soon as possible once a user mitigation is available.
+Once we get a good understanding of the vulnerability, we'll set a disclosure date after consulting the author of the report and Haystack maintainers.

VERSION.txt

@@ -0,0 +1 @@
+3.0.0-rc0

code_of_conduct.txt

@@ -0,0 +1,98 @@
+CONTRIBUTOR COVENANT CODE OF CONDUCT
+====================================
+
+Our Pledge
+----------
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for
+everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics,
+gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance,
+race, caste, color, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+Our Standards
+-------------
+
+Examples of behavior that contributes to a positive environment for our community include:
+    - Demonstrating empathy and kindness toward other people
+    - Being respectful of differing opinions, viewpoints, and experiences
+    - Giving and gracefully accepting constructive feedback
+    - Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
+    - Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior include:
+    - The use of sexualized language or imagery, and sexual attention or advances of any kind
+    - Trolling, insulting or derogatory comments, and personal or political attacks
+    - Public or private harassment
+    - Publishing others’ private information, such as a physical or email address, without their explicit permission
+    - Other conduct which could reasonably be considered inappropriate in a professional setting
+
+Enforcement Responsibilities
+----------------------------
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take
+appropriate and fair corrective action in response to any behavior that they deem inappropriate,
+threatening, offensive, or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits,
+issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for
+moderation decisions when appropriate.
+
+Scope
+-----
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially
+representing the community in public spaces. Examples of representing our community include using an official
+e-mail address, posting via an official social media account, or acting as an appointed representative
+at an online or offline event.
+
+Enforcement
+-----------
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible
+for enforcement at engage@deepset.ai. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the reporter of any incident.
+
+Enforcement Guidelines
+----------------------
+
+Community leaders will follow these Community Impact Guidelines in determining the consequences for any action
+they deem in violation of this Code of Conduct:
+
+1. Correction
+    Community Impact: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
+
+    Consequence: A private, written warning from community leaders, providing clarity around the nature of the violation
+    and an explanation of why the behavior was inappropriate. A public apology may be requested.
+
+2. Warning
+    Community Impact: A violation through a single incident or series of actions.
+
+    Consequence: A warning with consequences for continued behavior. No interaction with the people involved,
+    including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time.
+    This includes avoiding interactions in community spaces as well as external channels like social media.
+    Violating these terms may lead to a temporary or permanent ban.
+
+3. Temporary Ban
+    Community Impact: A serious violation of community standards, including sustained inappropriate behavior.
+
+    Consequence: A temporary ban from any sort of interaction or public communication with the community for a specified
+    period of time. No public or private interaction with the people involved, including unsolicited interaction with
+    those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
+
+4. Permanent Ban
+    Community Impact: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
+
+    Consequence: A permanent ban from any sort of public interaction within the community.
+
+Attribution
+-----------
+
+This Code of Conduct is adapted from the Contributor Covenant, version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by Mozilla’s code of conduct enforcement ladder.
+
+For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq.
+Translations are available at https://www.contributor-covenant.org/translations.

docker/Dockerfile.base

@@ -0,0 +1,35 @@
+# Pinned digest defaults so the base images are resolvable by hash even when
+# the args are not passed (e.g. by supply-chain scanners or a plain `docker build`).
+# docker/docker-bake.hcl overrides these at build time; keep the digests in sync.
+ARG build_image=python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203
+ARG base_image=python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203
+
+FROM ${build_image} AS build-image
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG haystack_version
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git
+
+COPY --from=ghcr.io/astral-sh/uv:0.11.17@sha256:03bdc89bb9798628846e60c3a9ad19006c8c3c724ccd2985a33145c039a0577b /uv /uvx /bin/
+
+# Shallow clone Haystack repo, we'll install from the local sources
+RUN git clone --depth=1 --branch=${haystack_version} https://github.com/deepset-ai/haystack.git /opt/haystack
+WORKDIR /opt/haystack
+
+# Use a virtualenv we can copy over the next build stage
+# Note: we use venv and not uv to create the virtualenv to make sure that the created virtualenv is accessible by pip
+# and prevent breaking changes in the image. uv can still be used to speed up installation.
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Upgrade setuptools due to https://nvd.nist.gov/vuln/detail/CVE-2022-40897
+RUN uv pip install --no-cache-dir -U setuptools && \
+    uv pip install --no-cache-dir .
+
+FROM ${base_image} AS final
+
+COPY --from=build-image /opt/venv /opt/venv
+
+ENV PATH="/opt/venv/bin:$PATH"

docker/README.md

@@ -0,0 +1,57 @@
+<p align="center">
+  <a href="https://haystack.deepset.ai/"><img src="https://raw.githubusercontent.com/deepset-ai/.github/main/haystack-logo-colored.png" alt="Haystack by deepset"></a>
+</p>
+
+[Haystack](https://github.com/deepset-ai/haystack) is an end-to-end LLM framework that allows you to build applications powered by LLMs, Transformer models, vector search and more. Whether you want to perform retrieval-augmented generation (RAG), document search, question answering or answer generation, Haystack can orchestrate state-of-the-art embedding models and LLMs into pipelines to build end-to-end NLP applications and solve your use case.
+
+## Haystack 2.x
+
+For the latest version of Haystack there's only one image available:
+
+- `haystack:base-<version>` contains a working Python environment with Haystack preinstalled. This image is expected to
+  be derived `FROM`.
+
+## Image Development
+
+Images are built with BuildKit and we use `bake` to orchestrate the process.
+You can build a specific image by running:
+```sh
+docker buildx bake base
+```
+
+You can override any `variable` defined in the `docker-bake.hcl` file and build custom
+images, for example if you want to use a branch from the Haystack repo, run:
+```sh
+HAYSTACK_VERSION=mybranch_or_tag BASE_IMAGE_TAG_SUFFIX=latest docker buildx bake base --no-cache
+```
+
+### Multi-Platform Builds
+
+Haystack images support multiple architectures. But depending on your operating system and Docker
+environment, you might not be able to build all of them locally.
+
+You may encounter the following error when trying to build the image:
+
+```
+multiple platforms feature is currently not supported for docker driver. Please switch to a different driver
+(eg. “docker buildx create --use”)
+```
+
+To get around this, you need to override the `platform` option and limit local builds to the same architecture as
+your computer's. For example, on an Apple M1 you can limit the builds to ARM only by invoking `bake` like this:
+
+```sh
+docker buildx bake base --set "*.platform=linux/arm64"
+```
+
+# License
+
+View [license information](https://github.com/deepset-ai/haystack/blob/main/LICENSE) for
+the software contained in this image.
+
+As with all Docker images, these likely also contain other software which may be under
+other licenses (such as Bash, etc from the base distribution, along with any direct or
+indirect dependencies of the primary software being contained).
+
+As for any pre-built image usage, it is the image user's responsibility to ensure that any
+use of this image complies with any relevant licenses for all software contained within.

docker/docker-bake.hcl

@@ -0,0 +1,40 @@
+variable "HAYSTACK_VERSION" {
+  default = "main"
+}
+
+variable "GITHUB_REF" {
+  default = ""
+}
+
+variable "IMAGE_NAME" {
+  default = "deepset/haystack"
+}
+
+variable "IMAGE_TAG_SUFFIX" {
+  default = "local"
+}
+
+variable "BASE_IMAGE_TAG_SUFFIX" {
+  default = "local"
+}
+
+variable "IS_STABLE" {
+  default = "false"
+}
+
+# 2.Y.Z releases are also tagged as "stable"
+# Example: 2.99.0 is tagged as base-2.99.0 and stable
+
+target "base" {
+  dockerfile = "Dockerfile.base"
+  tags = "${compact([
+    "${IMAGE_NAME}:base-${IMAGE_TAG_SUFFIX}",
+    equal("${IS_STABLE}", "true") ? "${IMAGE_NAME}:stable" : ""
+  ])}"
+  args = {
+    build_image = "python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203"
+    base_image = "python:3.12-slim@sha256:090ba77e2958f6af52a5341f788b50b032dd4ca28377d2893dcf1ecbdfdfe203"
+    haystack_version = "${HAYSTACK_VERSION}"
+  }
+  platforms = ["linux/amd64", "linux/arm64"]
+}

docs-website/.gitattributes

@@ -0,0 +1,3 @@
+* text=auto
+*.md text diff=markdown
+*.mdx text diff=markdown

docs-website/.gitignore

@@ -0,0 +1,52 @@
+.DS_Store
+.vscode/*
+!.vscode/extensions.json
+.idea
+*.iml
+*.code-workspace
+.changelog
+.history
+
+node_modules
+.yarn
+package-lock.json
+
+.eslintcache
+
+yarn-error.log
+website/build
+coverage
+.docusaurus
+.cache-loader
+types
+test-website
+test-website-in-workspace
+
+packages/create-docusaurus/lib/
+packages/lqip-loader/lib/
+packages/docusaurus/lib/
+packages/docusaurus-*/lib/*
+packages/eslint-plugin/lib/
+packages/stylelint-copyright/lib/
+
+website/netlifyDeployPreview/*
+website/changelog
+!website/netlifyDeployPreview/index.html
+!website/netlifyDeployPreview/_redirects
+
+website/_dogfooding/_swizzle_theme_tests
+
+CrowdinTranslations_*.zip
+
+website/.cpu-prof
+website/i18n/**/*
+.netlify
+
+build/
+
+# Auto-generated requirements files
+requirements.txt
+requirements-*.txt
+
+# Auto-generated llms.txt
+llms.txt

docs-website/CONTRIBUTING.md

@@ -0,0 +1,474 @@
+# Contributing to Haystack Documentation
+
+Thank you for your interest in contributing to the Haystack documentation! This guide provides everything you need to write, review, and maintain high-quality documentation for the Haystack project.
+
+This guide focuses specifically on documentation contributions. For code contributions, tests, or integrations in the main Haystack codebase, see the [main Haystack contribution guide](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md).
+
+## TL;DR — Your first docs PR in 10 minutes
+
+1. You won't be able to make changes directly to this repo, so the first step is to [create a fork](../CONTRIBUTING.md#clone-the-git-repository).
+
+2. Once your fork is ready, you can clone a local copy with:
+
+```bash
+git clone https://github.com/YOUR_USERNAME/haystack.git
+cd haystack/docs-website
+```
+
+3. Install and start:
+
+```bash
+npm install
+npm start
+```
+
+**Note:** All subsequent commands in this guide should be run from the `haystack/docs-website` directory unless otherwise specified.
+
+4. Edit under `docs/` for the unstable version, and under `versioned_docs/version-<highest>/` for the latest stable release. If you add a new page, include its ID in `sidebars.js` or the appropriate versioned sidebar.
+
+5. Optional production check:
+
+```bash
+npm run build && npm run serve
+```
+
+6. Commit and push:
+
+```bash
+git checkout -b docs/your-branch
+git add .
+git commit -m "docs: fix <desc>"
+git push -u origin HEAD
+```
+
+7. Open a PR and review the [Pull Request Checklist](#pull-request-checklist).
+
+**Table of Contents**
+
+- [TL;DR — Your first docs PR in 10 minutes](#tldr--your-first-docs-pr-in-10-minutes)
+- [Authoring New or Updated Pages](#authoring-new-or-updated-pages)
+  - [Where should I edit?](#where-should-i-edit)
+  - [Page Frontmatter](#page-frontmatter)
+  - [Updating Navigation](#updating-navigation)
+  - [Linking and Anchors](#linking-and-anchors)
+  - [Admonitions (Callouts)](#admonitions-callouts)
+- [Working with Templates](#working-with-templates)
+- [Testing](#testing)
+  - [Build Testing](#build-testing)
+- [API Reference Contributions](#api-reference-contributions)
+- [Understanding Documentation Versions and Where to Make Changes](#understanding-documentation-versions-and-where-to-make-changes)
+- [Preview Deployments](#preview-deployments)
+- [Troubleshooting](#troubleshooting)
+  - [Blank Page on npm start](#blank-page-on-npm-start)
+  - [Cache Issues](#cache-issues)
+  - [Build Errors](#build-errors)
+- [Moving or Removing Pages](#moving-or-removing-pages)
+- [Images and Assets](#images-and-assets)
+- [Pull Request Process](#pull-request-process)
+  - [Pull Request Checklist](#pull-request-checklist)
+- [Review Process](#review-process)
+- [Accessibility and Inclusivity](#accessibility-and-inclusivity)
+- [Getting Help](#getting-help)
+
+## Authoring New or Updated Pages
+
+### Where should I edit?
+
+| Your change | Edit here | Also edit here |
+|---|---|---|
+| New feature on Haystack `main` | `docs/` | — |
+| Fix in current stable docs | `docs/` | `versioned_docs/version-<highest>/` (for example, `version-2.20`) |
+| API reference content | Edit Python docstrings in main repo | — |
+
+### Page Frontmatter
+
+Every documentation page requires frontmatter at the top:
+
+```md
+---
+title: "Page Title"
+id: "page-id"
+description: "One to two sentences describing the page content for SEO and previews"
+slug: "/target-url"
+---
+```
+
+**Frontmatter fields:**
+
+- `title`: Displayed page title (title case)
+- `id`: Unique identifier for the page
+- `description`: SEO description (1-2 sentences)
+- `slug`: URL path for the page (optional, defaults to file path)
+
+### Updating Navigation
+
+After creating or moving a page, update the sidebar:
+
+**For narrative docs (`docs/`):**
+
+Edit `sidebars.js` and add your page to the appropriate category:
+
+```javascript
+{
+  type: 'category',
+  label: 'Concepts',
+  items: [
+    'concepts/pipelines',
+    'concepts/your-new-page',  // Add here
+  ],
+}
+```
+
+**For API reference (`reference/`):**
+
+Edit `reference-sidebars.js` if needed (however, most sections are auto-generated).
+
+### Linking and Anchors
+
+**Links within `docs/`:**
+
+Use relative paths for links within the same documentation section:
+
+```md
+See the [Pipeline Guide](../concepts/pipelines.mdx)
+See the [Components Overview](./components-overview.mdx)
+```
+
+**Links between `docs/` and `reference/`:**
+
+Because `docs/` and `reference/` are separate Docusaurus plugin instances, you must use absolute paths when linking between them:
+
+```md
+<!-- From docs/ to reference/ -->
+See the [Pipeline API Reference](/reference/haystack-api/pipelines/pipeline)
+
+<!-- From reference/ to docs/ -->
+See the [Pipeline Concepts Guide](/docs/concepts/pipelines)
+```
+
+**Note:** Always use `/docs/` or `/reference/` as the path prefix when linking across sections, not relative paths like `../../reference/`.
+
+**Explicit anchors:**
+
+For stable cross-links, use explicit heading IDs:
+
+```markdown
+## Installation {#install-guide}
+```
+
+Link to it: `[Install](./page.mdx#install-guide)` or `[Install](/docs/overview/quick-start#install-guide)` from `reference/`
+
+### Admonitions (Callouts)
+
+Use Docusaurus admonitions sparingly for supporting information:
+
+```mdx
+:::note[General notes or important information to highlight.]
+:::
+
+:::tip[Short tip that helps the reader succeed.]
+:::
+
+:::info[Useful but non-blocking background information.]
+:::
+
+:::warning[Risky settings or potential pitfalls.]
+:::
+
+:::danger[Data loss or security-impacting issues.]
+:::
+```
+
+## Working with Templates
+
+Starter templates are available in `docs/_templates/`:
+
+- `component-template.mdx` - For new component documentation
+- `document-store-template.mdx` - For new document store guides
+
+**How to use templates:**
+
+1. Copy the appropriate template from `docs/_templates/`
+2. Move the copy to its final location under `docs/`
+3. Update the frontmatter (title, id, description, slug)
+4. Fill in all sections marked with placeholders
+5. Update the sidebar to include your new page
+
+**Do not:**
+- Commit new documentation under `_templates/`
+- Leave template placeholder text in production docs
+
+## Testing
+
+### Build Testing
+
+We strongly recommend building the site locally before opening a PR:
+
+```bash
+npm run build
+```
+
+This command:
+- Builds production-ready static files
+- Validates all links and anchors
+- Reports broken links, duplicate routes, and errors
+
+**Fix all warnings before submitting your PR.** For minor changes like typo fixes, you may skip the local build and rely on CI feedback, but for substantial changes (new pages, restructuring, multiple edits), a local build helps catch issues early and saves CI time.
+
+## API Reference Contributions
+
+The API reference documentation is automatically generated from Python docstrings in the main Haystack codebase.
+
+**To update API documentation:**
+
+1. Edit docstrings in the [Haystack repository](https://github.com/deepset-ai/haystack)
+2. Open a PR in the main Haystack repo
+3. After merge, the API reference will be automatically synced through CI
+
+**Do not:**
+- Manually edit files in `reference/` or `reference_versioned_docs/`
+- Commit changes to auto-generated API documentation
+- Any manual changes will be overwritten by the next sync
+
+## Understanding Documentation Versions and Where to Make Changes
+
+The documentation structure supports multiple Haystack versions. Haystack releases new versions monthly, and documentation versioning is handled automatically through GitHub workflows during the release process.
+
+**Documentation directories:**
+- `docs/` - Unstable/next version (corresponds to Haystack's `main` branch)
+- `versioned_docs/version-X.Y/` - Stable release documentation for version X.Y
+
+**Note:** The highest version number in `versioned_docs/` represents the current stable release. For example, if you see `version-2.20`, `version-2.19`, and `version-2.18`, then version 2.20 is the current stable release, and older versions are for reference.
+
+**When to edit which version:**
+
+**Scenario 1: New feature or change in Haystack main branch**
+
+If you're documenting a new feature or change that exists in Haystack's `main` branch (next release):
+
+✅ Edit files in `docs/` (the unstable version)
+
+Example: A new component was added to Haystack main → document it in `docs/pipeline-components/`
+
+**Scenario 2: Bug fix or correction for current release**
+
+If you're fixing an error in the current release documentation (for example, incorrect information, broken link, typo):
+
+✅ Edit files in BOTH locations:
+1. `docs/` (so the fix persists in future versions)
+2. `versioned_docs/version-<highest>/` (the highest-numbered version directory)
+
+Example: A code example has a bug in the Pipelines guide → fix it in both `docs/concepts/pipelines.mdx` AND `versioned_docs/version-2.20/concepts/pipelines.mdx` (if 2.20 is the current stable release)
+
+**Pro tip:** When fixing bugs in current release docs, make the change in `docs/` first, then copy it to the highest-numbered versioned directory to ensure consistency.
+
+## Preview Deployments
+
+Pull requests that modify documentation will generate preview deployments once authorized by a maintainer. Once authorized, check your PR for a preview link, which allows you and reviewers to see the changes in a live environment before merging.
+
+Preview deployments include:
+- Full site build with your changes
+- All versions and navigation
+- Identical to production except for the URL
+
+## Troubleshooting
+
+### Blank Page on npm start
+
+If you see a blank page when running `npm start`:
+
+```bash
+# Clear Docusaurus cache
+npm run clear
+npm start
+```
+
+If the issue persists, build once to generate route metadata:
+
+```bash
+npm run build
+npm start
+```
+
+This is necessary because Docusaurus needs to generate internal routing metadata for versioned docs on first run.
+
+### Cache Issues
+
+Clear cached data if something looks off:
+
+```bash
+npm run clear
+```
+
+This removes:
+- `.docusaurus/` directory
+- Build cache
+- Generated metadata
+
+### Build Errors
+
+**Broken links:**
+- Check that all internal links use correct relative paths
+- Verify file names and paths match exactly (case-sensitive)
+- Ensure linked pages have proper frontmatter with `id` field
+
+**Duplicate routes:**
+- Check for duplicate `slug` values in frontmatter
+- Ensure no two pages map to the same URL path
+
+**Missing images:**
+- Verify image paths are correct
+- Check that images exist in `static/img/` or local `assets/` directories
+- Use relative paths from the markdown file location
+
+## Moving or Removing Pages
+
+**Moving a page:**
+
+1. Keep the existing URL stable by retaining the `slug` in frontmatter
+2. Update `sidebars.js` or `reference-sidebars.js` to reflect new file location
+3. Update any internal links that reference the moved page
+
+**Removing a page:**
+
+1. Remove the file from `docs/`
+2. Remove references from `sidebars.js`
+3. Check for and update any links pointing to the removed page
+4. Coordinate with maintainers for redirect setup if the URL was public
+
+**If a URL must change:**
+- Coordinate with maintainers to set up redirect rules
+- Avoid breaking inbound links from external sites
+
+## Images and Assets
+
+Shared images are stored in `static/img/`.
+
+**Best practices:**
+- Use descriptive filenames (for example, `pipeline-architecture.png`)
+- Optimize images before committing (use tools like ImageOptim, TinyPNG)
+- Prefer modern formats (WebP, optimized PNG/JPEG)
+- Always include alt text for accessibility
+
+**Adding images:**
+
+Use the `ClickableImage` component for all images. Import it at the top of your MDX file:
+
+```mdx
+import ClickableImage from "@site/src/components/ClickableImage";
+
+<ClickableImage
+  src="/img/pipeline-architecture.png"
+  alt="Pipeline architecture diagram"
+/>
+```
+
+**For zoomable images** (diagrams, screenshots that users may want to see in detail), use `size="large"`:
+
+```mdx
+<ClickableImage
+  src="/img/detailed-architecture.png"
+  alt="Detailed architecture diagram"
+  size="large"
+/>
+```
+
+**Images with transparent backgrounds:**
+
+For transparent PNGs that need better visibility in dark mode, add a background class:
+
+```mdx
+<!-- White background in dark mode -->
+<div className="img-white-bg">
+  <ClickableImage src="/img/logo.png" alt="Logo" />
+</div>
+
+<!-- Light grey background (softer) -->
+<div className="img-light-bg">
+  <ClickableImage src="/img/diagram.png" alt="Diagram" />
+</div>
+```
+
+**Youtube Embed**
+
+For embedding YouTube videos, use the `YouTubeEmbed` component:
+
+```mdx
+import YouTubeEmbed from "@site/src/components/YouTubeEmbed";
+
+<YouTubeEmbed videoId="dQw4w9WgXcQ" title="Example Video" />
+```
+
+## Pull Request Process
+
+### Pull Request Checklist
+
+Before submitting your PR, verify:
+
+- [ ] Content follows writing and style guidelines
+- [ ] Navigation updated (`sidebars.js` or `reference-sidebars.js`)
+- [ ] Internal links verified (no broken anchors)
+- [ ] Code samples tested and include language tags
+- [ ] Images optimized and include alt text
+- [ ] Local build passes (`npm run build`) - recommended for substantial changes
+- [ ] Vercel preview deployment succeeds (fix any deployment errors)
+- [ ] Conventional commit message format used in PR title
+- [ ] PR description includes context and related issues
+
+**PR title format:**
+
+Use conventional commits in the PR title:
+
+```
+docs: add troubleshooting guide for pipelines
+docs: fix typo in installation instructions
+docs: update API reference links
+```
+
+**PR description:**
+
+Include:
+- Summary of changes
+- Screenshots (if UI changes are visible)
+- Related issues (for example, "Fixes #123")
+- Testing performed
+- Notes for reviewers
+
+## Review Process
+
+1. Open a PR from your branch to `main`
+2. Automated checks will run (build validation)
+3. Maintainers will review your changes
+4. Address any requested changes
+5. Once approved and checks pass, a maintainer will merge
+6. Your changes will be deployed automatically
+
+## Accessibility and Inclusivity
+
+Ensure your documentation is accessible to all users:
+
+- **Alt text:** Provide descriptive alt text for all images
+- **Link text:** Use descriptive link text (not "click here")
+- **Language:** Use clear, concise sentences; avoid jargon where possible
+- **Examples:** Use inclusive language and diverse examples
+- **Headings:** Use proper heading hierarchy (don't skip levels)
+- **Code blocks:** Include language tags for proper syntax highlighting
+
+## Getting Help
+
+**Questions about contributing:**
+- Review this guide and the [main Haystack contribution guide](../CONTRIBUTING.md)
+- Check the [README](./README.md) for documentation site specifics
+- Check existing [issues](https://github.com/deepset-ai/haystack/issues) and [discussions](https://github.com/deepset-ai/haystack/discussions)
+- Ask in the [Discord community](https://discord.com/invite/haystack)
+
+**Technical issues:**
+- Search existing issues first
+- Open a new issue with the `documentation` label
+- Provide reproduction steps and environment details
+
+**Style or writing questions:**
+- Refer to the [Google Developer Documentation Style Guide](https://developers.google.com/style)
+- Ask maintainers for clarification in your PR
+
+Thank you for contributing to Haystack documentation! Your efforts help make Haystack more accessible and easier to use for everyone.

docs-website/README.md

@@ -0,0 +1,173 @@
+# Haystack Documentation Website
+
+This directory contains the Docusaurus-powered documentation website for [Haystack](https://github.com/deepset-ai/haystack), an open-source framework for building production-ready applications with Large Language Models (LLMs).
+
+- **Website URL:** https://docs.haystack.deepset.ai
+
+**Table of Contents**
+
+- [About](#about)
+- [Prerequisites](#prerequisites)
+- [Quick Start](#quick-start)
+- [Common tasks](#common-tasks)
+- [Project Structure](#project-structure)
+- [Technology Stack](#technology-stack)
+- [Available Scripts](#available-scripts)
+- [Contributing](#contributing)
+- [CI/CD and Automation](#cicd-and-automation)
+  - [Versioning](#versioning)
+- [Deployment](#deployment)
+- [llms.txt for AI tools](#llms.txt-for-ai-tools)
+
+## About
+
+This documentation site is built with Docusaurus 3 and provides comprehensive guides, tutorials, API references, and best practices for using Haystack. The site supports multiple versions and automated API reference generation.
+
+## Prerequisites
+
+- **Node.js** 18 or higher
+- **npm** (included with Node.js) or Yarn
+
+## Quick Start
+
+> [!NOTE]
+> All commands must be run from the `haystack/docs-website` directory.
+
+```bash
+# Clone the repository and navigate to docs-website
+git clone https://github.com/deepset-ai/haystack.git
+cd haystack/docs-website
+
+# Install dependencies
+npm install
+
+# Start the development server
+npm start
+
+# The site opens at http://localhost:3000 with live reload
+```
+
+## Common tasks
+
+- Edit a page: update files under `docs/` or `versioned_docs/` and preview at http://localhost:3000
+- Add to sidebar: update `sidebars.js` with your doc ID
+- Production check: `npm run build && npm run serve`
+- Full guidance: see `CONTRIBUTING.md`
+
+## Project Structure
+
+```
+docs-website/
+├── docs/                          # Main documentation (guides, tutorials, concepts)
+│   ├── _templates/               # Authoring templates (excluded from build)
+│   ├── concepts/                 # Core Haystack concepts
+│   ├── pipeline-components/      # Component documentation
+│   └── ...
+├── reference/                     # API reference (auto-generated, do not edit manually)
+├── versioned_docs/               # Versioned copies of docs/
+├── reference_versioned_docs/     # Versioned copies of reference/
+├── src/                          # React components and custom code
+│   ├── components/              # Custom React components
+│   ├── css/                     # Global styles
+│   ├── pages/                   # Custom pages
+│   ├── remark/                  # Remark plugins
+│   └── theme/                   # Docusaurus theme customizations
+├── static/                       # Static assets (images, files)
+├── scripts/                      # Build and test scripts
+│   ├── generate_requirements.py # Generates Python dependencies
+│   ├── setup-dev.sh             # Development environment setup
+│   └── test_python_snippets.py  # Tests Python code in docs
+├── sidebars.js                   # Navigation for docs/
+├── reference-sidebars.js         # Navigation for reference/
+├── docusaurus.config.js          # Main Docusaurus configuration
+├── versions.json                 # Available docs versions
+├── reference_versions.json       # Available API reference versions
+└── package.json                  # Node.js dependencies and scripts
+```
+
+## Technology Stack
+
+| Technology | Version | Purpose |
+|------------|---------|---------|
+| [Docusaurus](https://docusaurus.io/) | 3.8.1 | Static site generator |
+| [React](https://react.dev/) | 19.0.0 | UI framework |
+| [MDX](https://mdxjs.com/) | 3.0.0 | Markdown with JSX |
+| [Node.js](https://nodejs.org/) | ≥18.0 | Runtime environment |
+
+**Key Docusaurus Plugins:**
+- `@docusaurus/plugin-content-docs` — Two separate instances of this plugin run simultaneously:
+  1. **Main docs instance** (via the `classic` preset): serves `docs/` at `/docs/`
+  2. **Reference instance** (explicit plugin): serves `reference/` at `/reference/`
+
+  Each instance has its own sidebar, versioning config (`versions.json` vs `reference_versions.json`), and versioned content folders. This allows the API reference and guides to version independently and maintain separate navigation.
+
+- **Custom remark plugin** (`src/remark/versionedReferenceLinks.js`) — Automatically rewrites cross-links between docs and reference to include the correct version prefix. For example, if you're viewing docs version 2.19 and click a link to `/reference/some-api`, the plugin rewrites it to `/reference/2.19/some-api` so readers stay in the same version context.
+
+**When one might need these plugins:**
+- **Broken cross-links after a release:** If links between docs and API reference pages break (404s), the remark plugin may need adjustment—especially if version naming conventions change.
+- **Version dropdown issues:** If the version selector shows wrong versions or doesn't switch correctly between docs/reference, check the dual `plugin-content-docs` configs in `docusaurus.config.js`.
+- **Sidebar mismatches:** If API reference navigation breaks separately from main docs, remember they use different sidebar files (`sidebars.js` vs `reference-sidebars.js`).
+
+## Available Scripts
+
+**Important:** Run these commands from the `haystack/docs-website` directory:
+
+| Command | Description |
+|---------|-------------|
+| `npm install` | Install all dependencies |
+| `npm start` | Start development server with live reload (http://localhost:3000) |
+| `npm run build` | Build production-ready static files to `build/` |
+| `npm run serve` | Preview production build locally |
+| `npm run clear` | Clear Docusaurus cache (use if encountering build issues) |
+| `npm run docusaurus` | Run Docusaurus CLI commands directly |
+| `npm run swizzle` | Eject and customize Docusaurus theme components |
+
+## Contributing
+
+We welcome contributions to improve the documentation! See [CONTRIBUTING.md](./CONTRIBUTING.md) for:
+
+- Writing and style guidelines
+- How to author new documentation pages
+- Setting up your development environment
+- Testing requirements
+- Pull request process
+
+For code contributions to Haystack itself, see the [main repository's contribution guide](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md).
+
+## CI/CD and Automation
+
+This site uses automated workflows for API reference sync and preview deployments. See [CONTRIBUTING.md](./CONTRIBUTING.md) for details.
+
+### Versioning
+
+Documentation versions are released alongside Haystack releases and are fully automated through GitHub workflows. Contributors do not need to manually create or manage versions.
+
+**Automated Workflows:**
+- `promote_unstable_docs.yml` - Automatically triggered during Haystack releases
+- `minor_version_release.yml` - Creates new version directories and updates version configuration
+
+These workflows automatically create versioned documentation snapshots and pull requests during the release process.
+
+## Deployment
+
+The documentation site is automatically deployed to **https://docs.haystack.deepset.ai** when changes are merged to the `main` branch.
+
+## llms.txt for AI tools
+
+This docs site exposes a concatenated view of the documentation for AI tools with an `llms.txt` file, generated by the [`docusaurus-plugin-generate-llms-txt`](https://github.com/din0s/docusaurus-plugin-llms-txt) plugin.
+
+- **What it is**: A single, generated text file that concatenates the docs content to make it easier for LLMs and other tools to consume.
+- **Where to find it (deployed)**: At the site root `https://docs.haystack.deepset.ai/llms.txt`.
+- **How it’s generated**:
+  - Automatically when you run:
+    - `npm run start`
+    - `npm run build`
+  - Manually with:
+
+    ```bash
+    npm run generate-llms-txt
+    ```
+
+- **Configuration**:
+  - The plugin is wired in `docusaurus.config.js` under the `plugins` array as `'docusaurus-plugin-generate-llms-txt'` with `outputFile: 'llms.txt'`.
+  - A local plugin (`plugins/txtLoaderPlugin.js`) configures Webpack to treat `.txt` files (including `llms.txt`) as text assets so they don’t cause build-time parse errors.*

docs-website/api/mcp.ts

@@ -0,0 +1,72 @@
+// Files under `api/` become HTTP endpoints automatically — this is `/api/mcp`.
+import { VercelRequest, VercelResponse } from "@vercel/node";
+
+export default async function handler(req: VercelRequest, res: VercelResponse) {
+  res.setHeader("Access-Control-Allow-Origin", "*");
+  res.setHeader("Access-Control-Allow-Methods", "POST, OPTIONS");
+  res.setHeader(
+    "Access-Control-Allow-Headers",
+    "Content-Type, Accept, Mcp-Session-Id, Mcp-Protocol-Version"
+  );
+  res.setHeader("Access-Control-Expose-Headers", "Mcp-Session-Id");
+
+  if (req.method === "OPTIONS") {
+    return res.status(204).end();
+  }
+
+  if (req.method !== "POST") {
+    res.setHeader("Allow", "POST, OPTIONS");
+    return res.status(405).end("Method Not Allowed");
+  }
+
+  // Get environment variables
+  const { MCP_WORKSPACE_ID, SEARCH_API_TOKEN } = process.env;
+
+  if (!MCP_WORKSPACE_ID || !SEARCH_API_TOKEN) {
+    return res.status(500).json({ error: "MCP service is not configured." });
+  }
+
+  try {
+    // Forward the JSON-RPC body unchanged with the API key injected, so we
+    // don't need to know any MCP methods — new upstream tools just work.
+    const apiResponse = await fetch(
+      `https://api.cloud.deepset.ai/api/v2/workspaces/${MCP_WORKSPACE_ID}/mcp`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Accept:
+            (req.headers.accept as string) ||
+            "application/json, text/event-stream",
+          "X-Client-Source": "haystack-docs",
+          Authorization: `Bearer ${SEARCH_API_TOKEN}`,
+          // Forward MCP session id so upstream can correlate requests.
+          ...(req.headers["mcp-session-id"] && {
+            "Mcp-Session-Id": req.headers["mcp-session-id"] as string,
+          }),
+          // Forward the protocol version the client negotiated.
+          ...(req.headers["mcp-protocol-version"] && {
+            "Mcp-Protocol-Version": req.headers[
+              "mcp-protocol-version"
+            ] as string,
+          }),
+        },
+        body: JSON.stringify(req.body),
+      }
+    );
+
+    // Pass the response through as-is (status, content-type, raw body).
+    const text = await apiResponse.text();
+    res.status(apiResponse.status);
+    const contentType = apiResponse.headers.get("content-type");
+    if (contentType) res.setHeader("Content-Type", contentType);
+    // Surface the session id back to the client (browsers can read it because
+    // it's in Access-Control-Expose-Headers above).
+    const sessionId = apiResponse.headers.get("mcp-session-id");
+    if (sessionId) res.setHeader("Mcp-Session-Id", sessionId);
+    return res.send(text);
+  } catch (error) {
+    console.error("MCP proxy error:", error);
+    return res.status(502).json({ error: "Failed to reach MCP upstream." });
+  }
+}

docs-website/api/search.ts

@@ -0,0 +1,73 @@
+import { VercelRequest, VercelResponse } from "@vercel/node";
+
+export default async function handler(req: VercelRequest, res: VercelResponse) {
+  if (req.method !== "POST") {
+    res.setHeader("Allow", "POST");
+    return res.status(405).end("Method Not Allowed");
+  }
+
+  const { query, filter } = req.body;
+
+  if (!query) {
+    return res.status(400).json({ error: "Query is required" });
+  }
+
+  const { SEARCH_API_WORKSPACE, SEARCH_API_PIPELINE, SEARCH_API_TOKEN } =
+    process.env;
+
+  if (!SEARCH_API_WORKSPACE || !SEARCH_API_PIPELINE || !SEARCH_API_TOKEN) {
+    console.error(
+      "Search API environment variables are not configured on the server."
+    );
+    return res.status(500).json({ error: "Search service is not configured." });
+  }
+
+  try {
+    // Build the request body with optional filters
+    const requestBody: any = {
+      queries: [query],
+    };
+
+    // Add filters if provided (for future backend filtering support)
+    if (filter && filter !== "all") {
+      requestBody.debug = true;
+      requestBody.filters = {
+        operator: "AND",
+        conditions: [
+          {
+            field: "meta.type",
+            operator: "==",
+            value: filter,
+          },
+        ],
+      };
+    }
+
+    const apiResponse = await fetch(
+      `https://api.cloud.deepset.ai/api/v1/workspaces/${SEARCH_API_WORKSPACE}/pipelines/${SEARCH_API_PIPELINE}/search`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "X-Client-Source": "haystack-docs",
+          Authorization: `Bearer ${SEARCH_API_TOKEN}`,
+        },
+        body: JSON.stringify(requestBody),
+      }
+    );
+
+    if (!apiResponse.ok) {
+      const errorData = await apiResponse.text();
+      console.error("Haystack API error:", errorData);
+      return res
+        .status(apiResponse.status)
+        .json({ error: `API error: ${apiResponse.statusText}` });
+    }
+
+    const data = await apiResponse.json();
+    return res.status(200).json(data);
+  } catch (error) {
+    console.error("Internal server error:", error);
+    return res.status(500).json({ error: "Failed to fetch search results." });
+  }
+}

docs-website/api/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2021",
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "lib": ["ES2021", "DOM"],
+    "esModuleInterop": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true,
+    "types": ["node"]
+  },
+  "include": ["./**/*.ts"]
+}

docs-website/api/well-known.ts

@@ -0,0 +1,13 @@
+// Returns a JSON 404 for OAuth discovery probes (e.g.
+// `/.well-known/oauth-protected-resource`, `/.well-known/oauth-authorization-server`).
+// Per RFC 9728 / the MCP authorization spec a 404 here means "this resource
+// doesn't require OAuth — connect anonymously." Without this handler the
+// Docusaurus catch-all serves an HTML 404, which trips MCP clients that try to
+// JSON-parse the body to extract an OAuth error.
+import { VercelRequest, VercelResponse } from "@vercel/node";
+
+export default function handler(_req: VercelRequest, res: VercelResponse) {
+  res.setHeader("Content-Type", "application/json");
+  res.setHeader("Cache-Control", "public, max-age=3600");
+  return res.status(404).end("{}");
+}

docs-website/docs/_templates/component-template.mdx

@@ -0,0 +1,44 @@
+---
+title: "Component Name"
+id: "component-name"
+description: "A short description of the component"
+slug: "/component-name"
+---
+
+# Component Name
+
+<div className="key-value-table">
+
+|  |  |
+| --- | --- |
+| **Most common position in a pipeline** |  |
+| **Mandatory init variables** |  |
+| **Mandatory run variables** |  |
+| **Output variables** |  |
+| **API reference** |  |
+| **GitHub link** |  |
+| **Package name** |  |
+
+</div>
+
+## Overview
+
+*What does it do in general? For example,..?*
+
+*How does it work more specifically? Are there any pitfalls to pay attention to?*
+
+*(if applicable) How is it different from this other very similar component? Which one do you choose?*
+
+## Usage
+
+*Any mandatory imports?*
+
+### On its own
+
+*Code snippet on how to run a component*
+
+### In a pipeline
+
+*Code snippet of a component being introduced in a pipeline*
+
+*There can be more than one example. Add examples of pipelines where this component would be most useful, for example RAG, doc retrieval, etc.*

docs-website/docs/_templates/document-store-template.mdx

@@ -0,0 +1,30 @@
+---
+title: "Document Store Name"
+id: "document-store-name"
+description: "A short description of the document store"
+slug: "/document-store-name"
+---
+
+# Document Store Name
+
+## Description
+
+*What are this Document Store features? When would a user select it, and when not?*
+
+*Are there any limitations?*
+
+*Users are often curious to know if a document store supports metadata filtering and sparse vectors.*
+
+## Initialization
+
+*Describe how to get this Document Store to work, with code samples.*
+
+## Supported Retrievers
+
+*Name of the supported Retriever(s).*
+
+*If several – describe how to choose an appropriate one for user’s goals (perhaps, one is faster and the other is more accurate).*
+
+## Link to GitHub
+
+*for example [https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/gradient](https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/gradient)*

docs-website/docs/concepts/agents.mdx

@@ -0,0 +1,127 @@
+---
+title: "Agents"
+id: agents
+slug: "/agents"
+description: "This page explains how to create an AI agent in Haystack capable of retrieving information, generating responses, and taking actions using various Haystack components."
+---
+
+# Agents
+
+This page explains how to create an AI agent in Haystack capable of retrieving information, generating responses, and taking actions using various Haystack components.
+
+## What’s an AI Agent?
+
+An AI agent is a system that can:
+
+- Understand user input (text, image, audio, and other queries),
+- Retrieve relevant information (documents or structured data),
+- Generate intelligent responses (using LLMs like OpenAI or Hugging Face models),
+- Perform actions (calling APIs, fetching live data, executing functions).
+
+AI agents are autonomous systems that use large language models (LLMs) to make decisions and solve complex tasks.
+They interact with their environment using tools, memory, and reasoning.
+An AI agent is more than a chatbot — it actively plans, chooses the right tools, and executes tasks to achieve a goal.
+Unlike traditional software, it adapts to new information and refines its process as needed.
+
+1. **LLM as the Brain**: The agent’s core is an LLM, which understands context, processes natural language and serves as the central intelligence system.
+2. **Tools for Interaction**: Agents connect to external tools, APIs, and databases to gather information and take action.
+3. **Memory for Context**: Short-term memory helps track conversations, while long-term memory stores knowledge for future interactions.
+4. **Reasoning and Planning**: Agents break down complex problems, come up with step-by-step action plans, and adapt based on new data and feedback.
+
+An AI agent starts with a prompt that defines its role and objectives.
+It decides when to use tools, gathers data, and refines its approach through loops of reasoning and action.
+For example, a customer service agent answers queries using a database — if it lacks an answer, it fetches real-time data, summarizes it, and responds.
+A coding assistant understands project requirements, suggests solutions, and writes code.
+
+## Key Components
+
+### Agent Component
+
+Haystack has a built-in [Agent](../pipeline-components/agents-1/agent.mdx) component that manages the full tool-calling loop — it calls the LLM, invokes tools, updates state, and continues until a stopping condition is met.
+Key capabilities include:
+
+- **State management**: Share typed data between tools, accumulate results across iterations, and surface them in the result dict using `state_schema`. See [State](../pipeline-components/agents-1/state.mdx).
+- **Streaming**: Stream token-by-token output with a `streaming_callback`.
+- **Human-in-the-loop**: Intercept tool calls for human review before execution. See [Human in the Loop](../pipeline-components/agents-1/human-in-the-loop.mdx).
+- **Multi-agent systems**: Wrap an `Agent` as a `ComponentTool` to build coordinator/specialist architectures. See [Multi-Agent Systems](./agents/multi-agent-systems.mdx).
+- **MCP server exposure**: Expose your agent as an MCP server using [Hayhooks](../development/hayhooks.mdx), making it callable from any MCP-compatible client such as Claude Desktop or Cursor.
+- **Multimodal inputs**: Pass images alongside text using `ImageContent` in `ChatMessage` content parts, or return `ImageContent` from tools for dynamic image analysis. Requires a vision-capable model such as `gpt-5` or `gemini-2.5-flash`. See [Multimodal Inputs](../pipeline-components/agents-1/agent.mdx#multimodal-inputs).
+
+Check out the [Agent](../pipeline-components/agents-1/agent.mdx) documentation, or the [example](#tool-calling-agent) below to get started.
+
+### State
+
+[`State`](../pipeline-components/agents-1/state.mdx) is Haystack's built-in mechanism for sharing data between tools and accumulating results across multiple tool calls.
+You define a `state_schema` on the `Agent`, and any keys declared there are returned alongside `messages` and `last_message` in the agent's result dict.
+
+### Tools
+
+Haystack provides several ways to create and manage tools:
+
+- [`Tool`](../tools/tool.mdx) class / [`@tool`](../tools/tool.mdx#tool-decorator) decorator – Define a tool from a Python function. The `@tool` decorator automatically uses the function's name and docstring; the `Tool` class gives full control over the name, description, and schema.
+- [`ComponentTool`](../tools/componenttool.mdx) – Wraps any Haystack component as a callable tool.
+- [`PipelineTool`](../tools/pipelinetool.mdx) – Wraps a full Haystack pipeline as a callable tool.
+- [`MCPTool`](../tools/mcptool.mdx) / [`MCPToolset`](../tools/mcptoolset.mdx) – Connects to Model Context Protocol (MCP) servers to load external tools.
+- [`Toolset`](../tools/toolset.mdx) – Groups multiple tools into a single unit to pass to an Agent or Generator.
+- [`SearchableToolset`](../tools/searchabletoolset.mdx) – Enables keyword-based tool discovery for large catalogs, so the LLM only sees relevant tools at each step.
+
+## Example
+
+### Tool-Calling Agent
+
+Create a tool-calling agent with the `Agent` component. This example requires `OPENAI_API_KEY` and `SERPERDEV_API_KEY` to be set as environment variables:
+
+```shell
+export OPENAI_API_KEY=<your-openai-key>
+export SERPERDEV_API_KEY=<your-serperdev-key>
+```
+
+The examples on this page use SerperDev web search component that have moved to the `serperdev-haystack` package. Install it to run the examples:
+
+```shell
+pip install serperdev-haystack
+```
+
+```python
+from haystack.components.agents import Agent
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.generators.utils import print_streaming_chunk
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
+from haystack.dataclasses import ChatMessage
+from haystack.tools import ComponentTool
+
+# Wrap the web search component as a tool
+web_tool = ComponentTool(
+    component=SerperDevWebSearch(top_k=3),
+    name="web_search",
+    description="Search the web for current information like weather, news, or facts.",
+)
+
+tool_calling_agent = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    system_prompt=(
+        "You're a helpful agent. When asked about current information like weather, news, or facts, "
+        "use the web_search tool to find the information and then summarize the findings."
+    ),
+    tools=[web_tool],
+    streaming_callback=print_streaming_chunk,
+)
+
+result = tool_calling_agent.run(
+    messages=[ChatMessage.from_user("How is the weather in Berlin?")],
+)
+print(result["last_message"].text)
+```
+
+Resulting in:
+
+```python
+>>> The current weather in Berlin is approximately 60°F. The forecast for today includes clouds in the morning with some sunshine later. The high temperature is expected to be around 65°F, and the low tonight will drop to 40°F.
+
+- **Morning**: 49°F
+- **Afternoon**: 57°F
+- **Evening**: 47°F
+- **Overnight**: 39°F
+
+For more details, you can check the full forecasts on [AccuWeather](https://www.accuweather.com/en/de/berlin/10178/current-weather/178087) or [Weather.com](https://weather.com/weather/today/l/5ca23443513a0fdc1d37ae2ffaf5586162c6fe592a66acc9320a0d0536be1bb9).
+```

docs-website/docs/concepts/agents/multi-agent-systems.mdx

@@ -0,0 +1,346 @@
+---
+title: "Multi-Agent Systems"
+id: multi-agent-systems
+slug: "/multi-agent-systems"
+description: "Learn how to build multi-agent systems in Haystack by spawning agents as tools. Use the @tool decorator or ComponentTool to connect specialist agents to a coordinator."
+---
+
+# Multi-Agent Systems
+
+Multi-agent systems let you compose multiple `Agent` instances into larger architectures where a **coordinator** agent delegates to **specialist** agents.
+Each specialist focuses on a specific task with its own tools and system prompt - the coordinator plans and routes work without needing to know how each task gets done.
+
+Spawning agents as tools is useful when:
+
+- A task is too broad for a single agent to handle reliably,
+- You want to isolate different capabilities into focused, reusable agents,
+- You need to keep the coordinator's context lean for better decisions and lower token usage.
+
+In Haystack, you spawn a specialist agent as a tool using either the `@tool` decorator (recommended) or `ComponentTool`.
+
+## Converting an Agent to a Tool
+
+### `@tool` Decorator (Recommended)
+
+Wrapping an agent inside a `@tool` function gives you full control over what the coordinator LLM sees:
+
+- **Simplified parameters**: define explicit `Annotated` arguments instead of exposing `agent.run()`'s full interface
+- **Formatted output**: extract and return only what the coordinator needs, rather than the full result dict
+- **Error handling**: catch exceptions and return a clean message so the coordinator can recover
+
+This approach works better with smaller LLMs because the tool has a clean, minimal signature.
+The coordinator only needs to provide a query string - all the `ChatMessage` construction and result unpacking is hidden inside the function.
+
+The examples on this page use SerperDev web search component that have moved to the `serperdev-haystack` package. Install it to run the examples:
+
+```shell
+pip install serperdev-haystack
+```
+
+```python
+from typing import Annotated
+from haystack.components.agents import Agent
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.generators.utils import print_streaming_chunk
+from haystack.dataclasses import ChatMessage
+from haystack.tools import ComponentTool, tool
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
+from haystack.utils import Secret
+
+
+research_agent = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    tools=[
+        ComponentTool(
+            component=SerperDevWebSearch(
+                api_key=Secret.from_env_var("SERPERDEV_API_KEY"),
+                top_k=3,
+            ),
+            name="web_search",
+            description="Search the web for current information on any topic",
+        ),
+    ],
+    system_prompt="You are a research specialist. Search the web to find information.",
+)
+
+
+@tool
+def research(query: Annotated[str, "The research question to investigate"]) -> str:
+    """Research a topic and return a summary of findings."""
+    try:
+        result = research_agent.run(messages=[ChatMessage.from_user(query)])
+        return result["last_message"].text
+    except Exception as e:
+        return f"Research failed: {e}"
+
+
+coordinator = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    tools=[research],
+    system_prompt="You are a coordinator. Delegate research tasks to the research tool.",
+    streaming_callback=print_streaming_chunk,
+)
+
+result = coordinator.run(
+    messages=[
+        ChatMessage.from_user("What are the latest developments in Haystack AI?"),
+    ],
+)
+```
+
+### `ComponentTool`
+
+`ComponentTool` wraps an agent directly without a wrapper function.
+Choose it when you want **declarative configuration**: the full specialist setup (model, tools, system prompt) lives in one serializable object alongside the coordinator.
+
+Use `outputs_to_string={"source": "last_message"}` to surface only the specialist's final reply to the coordinator rather than the full result dict.
+
+```python
+from haystack.tools import ComponentTool
+
+research_tool = ComponentTool(
+    component=research_agent,
+    name="research_specialist",
+    description="A specialist that researches topics on the web",
+    outputs_to_string={"source": "last_message"},
+)
+
+coordinator = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    tools=[research_tool],
+    system_prompt="You are a coordinator. Delegate research tasks to the research specialist.",
+    streaming_callback=print_streaming_chunk,
+)
+
+result = coordinator.run(
+    messages=[
+        ChatMessage.from_user("What are the latest developments in Haystack AI?"),
+    ],
+)
+```
+
+The full specialist configuration is captured inline when serialized.
+Wrap the coordinator in a `Pipeline` and call `pipeline.dumps()` to get the YAML, which can be loaded back with `Pipeline.loads()`.
+
+<details>
+<summary>View YAML</summary>
+
+```yaml
+components:
+  coordinator:
+    init_parameters:
+      chat_generator:
+        init_parameters:
+          api_base_url: null
+          api_key:
+            env_vars:
+            - OPENAI_API_KEY
+            strict: true
+            type: env_var
+          generation_kwargs: {}
+          http_client_kwargs: null
+          max_retries: null
+          model: gpt-5.4-nano
+          organization: null
+          streaming_callback: null
+          timeout: null
+          tools: null
+          tools_strict: false
+        type: haystack.components.generators.chat.openai.OpenAIChatGenerator
+      exit_conditions:
+      - text
+      hooks: null
+      max_agent_steps: 100
+      raise_on_tool_invocation_failure: false
+      required_variables: null
+      state_schema: {}
+      streaming_callback: null
+      system_prompt: You are a coordinator. Delegate research tasks to the research
+        specialist. Keep your final answer concise.
+      tool_concurrency_limit: 4
+      tool_streaming_callback_passthrough: false
+      tools:
+      - data:
+          component:
+            init_parameters:
+              chat_generator:
+                init_parameters:
+                  api_base_url: null
+                  api_key:
+                    env_vars:
+                    - OPENAI_API_KEY
+                    strict: true
+                    type: env_var
+                  generation_kwargs: {}
+                  http_client_kwargs: null
+                  max_retries: null
+                  model: gpt-5.4-nano
+                  organization: null
+                  streaming_callback: null
+                  timeout: null
+                  tools: null
+                  tools_strict: false
+                type: haystack.components.generators.chat.openai.OpenAIChatGenerator
+              exit_conditions:
+              - text
+              hooks: null
+              max_agent_steps: 100
+              raise_on_tool_invocation_failure: false
+              required_variables: null
+              state_schema: {}
+              streaming_callback: null
+              system_prompt: You are a research specialist. Search the web to find
+                information. Return a concise summary of your findings in 3-5 sentences.
+              tool_concurrency_limit: 4
+              tool_streaming_callback_passthrough: false
+              tools:
+              - data:
+                  component:
+                    init_parameters:
+                      allowed_domains: null
+                      api_key:
+                        env_vars:
+                        - SERPERDEV_API_KEY
+                        strict: true
+                        type: env_var
+                      exclude_subdomains: false
+                      search_params: {}
+                      top_k: 3
+                    type: haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch
+                  description: Search the web for current information on any topic
+                  inputs_from_state: null
+                  name: web_search
+                  outputs_to_state: null
+                  outputs_to_string: null
+                  parameters: null
+                type: haystack.tools.component_tool.ComponentTool
+              user_prompt: null
+            type: haystack.components.agents.agent.Agent
+          description: A specialist that researches topics on the web
+          inputs_from_state: null
+          name: research_specialist
+          outputs_to_state: null
+          outputs_to_string:
+            source: last_message
+          parameters: null
+        type: haystack.tools.component_tool.ComponentTool
+      user_prompt: null
+    type: haystack.components.agents.agent.Agent
+connection_type_validation: true
+connections: []
+max_runs_per_component: 100
+metadata: {}
+```
+
+</details>
+
+## Coordinator / Specialist Pattern
+
+The coordinator/specialist pattern cleanly splits responsibilities: the coordinator handles planning and delegation, while each specialist owns a focused toolset and a targeted system prompt.
+
+This is also a form of **context engineering**: deliberately controlling what each agent sees.
+A specialist accumulates its own tool call trace, but the coordinator only needs the final answer.
+Returning just `result["last_message"].text` (with `@tool`) or using `outputs_to_string` (with `ComponentTool`) surfaces only the specialist's final reply, keeping the coordinator's context lean.
+
+When covering multiple topics, the coordinator can call the same specialist tool several times in a single response.
+All tool calls from one LLM response are executed concurrently using a thread pool.
+Control the level of parallelism with the `tool_concurrency_limit` init parameter (default: `4`).
+
+The example below asks the coordinator about two topics: it calls `research` twice and both specialists run in parallel.
+
+`HTMLToDocument` uses [Trafilatura](https://trafilatura.readthedocs.io) to extract clean text from HTML pages.
+Install it before running:
+
+```shell
+pip install trafilatura
+```
+
+```python
+from typing import Annotated
+from haystack.components.agents import Agent
+from haystack.components.converters import HTMLToDocument
+from haystack.components.fetchers.link_content import LinkContentFetcher
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.generators.utils import print_streaming_chunk
+from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
+from haystack.dataclasses import ChatMessage
+from haystack.tools import ComponentTool, tool
+from haystack.utils import Secret
+
+
+search_tool = ComponentTool(
+    component=SerperDevWebSearch(
+        api_key=Secret.from_env_var("SERPERDEV_API_KEY"),
+        top_k=3,
+    ),
+    name="web_search",
+    description="Search the web for current information on any topic",
+)
+
+
+@tool
+def fetch_page(url: Annotated[str, "The URL of the web page to fetch"]) -> str:
+    """Fetch the content of a web page given its URL."""
+    try:
+        streams = LinkContentFetcher().run(urls=[url])["streams"]
+        if not streams:
+            return "No content found."
+        documents = HTMLToDocument().run(sources=streams)["documents"]
+        return documents[0].content if documents else "No content extracted."
+    except Exception as e:
+        return f"Failed to fetch page: {e}"
+
+
+research_agent = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    tools=[search_tool, fetch_page],
+    system_prompt=(
+        "You are a research specialist. Search the web to find relevant pages, "
+        "then fetch their full content for detailed information. "
+        "Return a concise summary of your findings in 3-5 sentences."
+    ),
+)
+
+
+@tool
+def research(query: Annotated[str, "The research question to investigate"]) -> str:
+    """Research a topic and return a summary of findings."""
+    try:
+        result = research_agent.run(messages=[ChatMessage.from_user(query)])
+        return result["last_message"].text
+    except Exception as e:
+        return f"Research failed: {e}"
+
+
+coordinator = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-5.4-nano"),
+    tools=[research],
+    system_prompt=(
+        "You are a coordinator. Delegate research tasks to the research tool. "
+        "For questions covering multiple topics, research each one independently. "
+        "Keep your final answer concise."
+    ),
+    streaming_callback=print_streaming_chunk,
+    tool_concurrency_limit=4,  # run up to 4 specialist calls in parallel
+)
+
+result = coordinator.run(
+    messages=[
+        ChatMessage.from_user(
+            "What are the latest developments in large language models and retrieval-augmented generation?",
+        ),
+    ],
+)
+```
+
+## Additional References
+
+📖 Related docs:
+
+- [Agent](../../pipeline-components/agents-1/agent.mdx)
+- [State](../../pipeline-components/agents-1/state.mdx)
+- [ComponentTool](../../tools/componenttool.mdx)
+
+📚 Tutorials:
+
+- [Creating a Multi-Agent System](https://haystack.deepset.ai/tutorials/45_creating_a_multi_agent_system)

docs-website/docs/concepts/components.mdx

@@ -0,0 +1,67 @@
+---
+title: "Components"
+id: components
+slug: "/components"
+description: "Components are the building blocks of a pipeline. They perform tasks such as preprocessing, retrieving, or summarizing text while routing queries through different branches of a pipeline. This page is a summary of all component types available in Haystack."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+
+# Components
+
+Components are the building blocks of a pipeline. They perform tasks such as preprocessing, retrieving, or summarizing text while routing queries through different branches of a pipeline. This page is a summary of all component types available in Haystack.
+
+Components are connected to each other using a [pipeline](pipelines.mdx), and they function like building blocks that can be easily switched out for each other. A component can take the selected outputs of other components as input. You can also provide input to a component when you call `pipeline.run()`.
+
+## Stand-Alone or In a Pipeline
+
+You can integrate components in a pipeline to perform a specific task. But you can also use some of them stand-alone, outside of a pipeline. For example, you can run `DocumentWriter` on its own, to write documents into a Document Store. To check how to use a component and if it's usable outside of a pipeline, check the _Usage_ section on the component's documentation page.
+
+Each component has a `run()` method. When you connect components in a pipeline, and you run the pipeline by calling `Pipeline.run()`, it invokes the `run()` method for each component sequentially.
+
+## Input and Output
+
+To connect components in a pipeline, you need to know the names of the inputs and outputs they accept. The output of one component must be compatible with the input the subsequent component accepts. For example, to connect Retriever and Ranker in a pipeline, you must know that the Retriever outputs `documents` and the Ranker accepts `documents` as input.
+
+The mandatory inputs and outputs are listed in a table at the top of each component's documentation page so that you can quickly check them:
+<ClickableImage src="/img/3a53f3e-inputs_and_outputs.png" alt="DocumentWriter component specification table showing Name, Folder Path, Position in Pipeline, Inputs (documents list), and Outputs (documents_written integer)" />
+
+You can also look them up in the code in the component`run()` method. Here's an example of the inputs and outputs of `TransformerSimilarityRanker`:
+
+```python
+@component.output_types(documents=List[Document]) # "documents" is the output name you need when connecting components in a pipeline
+def run(self, query: str, documents: List[Document], top_k: Optional[int] = None):# "query" and "documents" are the mandatory inputs, additionally you can also specify the optional top_k parameter
+"""
+Returns a list of Documents ranked by their similarity to the given query.
+
+:param query: Query string.
+:param documents: List of Documents.
+:param top_k: The maximum number of Documents you want the Ranker to return.
+:return: List of Documents sorted by their similarity to the query with the most similar Documents appearing first.
+"""
+```
+
+## Warming Up Components
+
+Components that use heavy resources, like LLMs or embedding models, have a `warm_up()` method that loads the necessary resources (such as models) into memory. This method is automatically called the first time the component runs, so you can use components directly without explicitly calling `warm_up()`:
+
+The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples:
+
+```shell
+pip install sentence-transformers-haystack
+```
+
+```python
+from haystack import Document
+from haystack_integrations.components.embedders.sentence_transformers import (
+    SentenceTransformersDocumentEmbedder,
+)
+
+doc = Document(content="I love pizza!")
+doc_embedder = SentenceTransformersDocumentEmbedder()
+
+result = doc_embedder.run([doc])  # warm_up() is called automatically on first run
+print(result["documents"][0].embedding)
+```
+
+You can still call `warm_up()` explicitly if you want to control when resources are loaded.

docs-website/docs/concepts/components/custom-components.mdx

@@ -0,0 +1,182 @@
+---
+title: "Creating Custom Components"
+id: custom-components
+slug: "/custom-components"
+description: "Create your own components and use them standalone or in pipelines."
+---
+
+# Creating Custom Components
+
+Create your own components and use them standalone or in pipelines.
+
+With Haystack, you can easily create any custom components for various tasks, from filtering results to integrating with external software. You can then insert, reuse, and share these components within Haystack or even with an external audience by packaging them and submitting them to [Haystack Integrations](../integrations.mdx)!
+
+## Requirements
+
+Here are the requirements for all custom components:
+
+- `@component`: This decorator marks a class as a component, allowing it to be used in a pipeline.
+- `run()`: This is a required method in every component. It accepts input arguments and returns a `dict`. The inputs can either come from the pipeline when it’s executed, or from the output of another component when connected using `connect()`. The `run()` method should be compatible with the input/output definitions declared for the component. See an [Extended Example](#extended-example) below to check how it works.
+
+
+:::note[Avoid in-place input mutation]
+
+When building custom components, do not change the component's inputs directly. Instead, work on a copy or a new version of the input, modify that, and return it. The reason for this is that the original input values might be reused by other components or by later pipeline steps. Mutating the input directly can lead to unintended side effects and bugs in the pipeline, as other components might rely on the original input values.
+
+When only one or a few fields of the input need to be changed (for example, `meta` on a `Document`), use `dataclasses.replace()` to create a new instance with the updated fields. This is simpler and more efficient than deep-copying the whole object:
+
+```python
+from dataclasses import replace
+
+
+def run(self, documents):
+    updated = [replace(doc, meta={**doc.meta, "processed": True}) for doc in documents]
+    return {"documents": updated}
+```
+
+When you need to modify nested mutable structures, for example `list` or `dict` attributes, or update many fields of the dataclass instance, use a full deep copy instead:
+
+```python
+import copy
+
+
+def run(self, documents):
+    documents_copy = copy.deepcopy(documents)
+    # mutate documents_copy safely here
+    return {"documents": documents_copy}
+```
+
+:::
+
+
+### Inputs and Outputs
+
+Next, define the inputs and outputs for your component.
+
+#### Inputs
+
+You can choose between three input options:
+
+- `set_input_type`: This method defines or updates a single input socket for a component instance. It’s ideal for adding or modifying a specific input at runtime without affecting others. Use this when you need to dynamically set or modify a single input based on specific conditions.
+- `set_input_types`: This method allows you to define multiple input sockets at once, replacing any existing inputs. It’s useful when you know all the inputs the component will need and want to configure them in bulk. Use this when you want to define multiple inputs during initialization.
+- Declaring arguments directly in the `run()` method. Use this method when the component’s inputs are static and known at the time of class definition.
+
+#### Outputs
+
+You can choose between two output options:
+
+- `@component.output_types`: This decorator defines the output types and names at the time of class definition. The output names and types must match the `dict` returned by the `run()` method. Use this when the output types are static and known in advance. This decorator is cleaner and more readable for static components.
+- `set_output_types`: This method defines or updates multiple output sockets for a component instance at runtime. It’s useful when you need flexibility in configuring outputs dynamically. Use this when the output types need to be set at runtime for greater flexibility.
+
+## Short Example
+
+Here is an example of a simple minimal component setup:
+
+```python
+from haystack import component
+
+
+@component
+class WelcomeTextGenerator:
+    """
+    A component generating personal welcome message and making it upper case
+    """
+
+    @component.output_types(welcome_text=str, note=str)
+    def run(self, name: str):
+        return {
+            "welcome_text": f"Hello {name}, welcome to Haystack!".upper(),
+            "note": "welcome message is ready",
+        }
+```
+
+Here, the custom component `WelcomeTextGenerator` accepts one input: `name` string and returns two outputs: `welcome_text` and `note`.
+
+## Extended Example
+
+Check out an example below on how to create two custom components and connect them in a Haystack pipeline.
+
+```python
+# import necessary dependencies
+from haystack import component, Pipeline
+
+
+# Create two custom components. Note the mandatory @component decorator and @component.output_types, as well as the mandatory run method.
+@component
+class WelcomeTextGenerator:
+    """
+    A component generating personal welcome message and making it upper case
+    """
+
+    @component.output_types(welcome_text=str, note=str)
+    def run(self, name: str):
+        return {
+            "welcome_text": (
+                "Hello {name}, welcome to Haystack!".format(name=name)
+            ).upper(),
+            "note": "welcome message is ready",
+        }
+
+
+@component
+class WhitespaceSplitter:
+    """
+    A component for splitting the text by whitespace
+    """
+
+    @component.output_types(split_text=list[str])
+    def run(self, text: str):
+        return {"split_text": text.split()}
+
+
+# create a pipeline and add the custom components to it
+text_pipeline = Pipeline()
+text_pipeline.add_component(
+    name="welcome_text_generator",
+    instance=WelcomeTextGenerator(),
+)
+text_pipeline.add_component(name="splitter", instance=WhitespaceSplitter())
+
+# connect the components
+text_pipeline.connect(
+    sender="welcome_text_generator.welcome_text",
+    receiver="splitter.text",
+)
+
+# define the result and run the pipeline
+result = text_pipeline.run({"welcome_text_generator": {"name": "Bilge"}})
+
+print(result["splitter"]["split_text"])
+```
+
+## Extending the Existing Components
+
+To extend already existing components in Haystack, subclass an existing component and use the `@component` decorator to mark it. Override or extend the `run()` method to process inputs and outputs. Call `super()` with the derived class name from the init of the derived class to avoid initialization issues:
+
+```python
+class DerivedComponent(BaseComponent):
+    def __init__(self):
+        super(DerivedComponent, self).__init__()
+
+
+# ...
+
+dc = DerivedComponent()  # ok
+```
+
+An example of an extended component is Haystack's [FaithfulnessEvaluator](https://github.com/deepset-ai/haystack/blob/e5a80722c22c59eb99416bf0cd712f6de7cd581a/haystack/components/evaluators/faithfulness.py) derived from LLMEvaluator.
+
+## Project Template
+
+If you're building a custom component that you want to package and share, we provide a [GitHub template repository](https://github.com/deepset-ai/custom-component) that gives you a ready-made project structure. It includes the boilerplate for packaging, testing, and distributing your custom component as a standalone Python package. Use it to quickly scaffold a new integration or reusable component without setting up the project from scratch.
+
+Check out the [video walkthrough](https://www.youtube.com/watch?v=SWC0QecAMcI) for a step-by-step guide on how to use the template.
+
+## Additional References
+
+🧑‍🍳 Cookbooks:
+
+- [Build quizzes and adventures with Character Codex and llamafile](https://haystack.deepset.ai/cookbook/charactercodex_llamafile/)
+- [Run tasks concurrently within a custom component](https://haystack.deepset.ai/cookbook/concurrent_tasks/)
+- [Chat With Your SQL Database](https://haystack.deepset.ai/cookbook/chat_with_sql_3_ways/)
+- [Hacker News Summaries with Custom Components](https://haystack.deepset.ai/cookbook/hackernews-custom-component-rag/)

docs-website/docs/concepts/components/supercomponents.mdx

@@ -0,0 +1,195 @@
+---
+title: "SuperComponents"
+id: supercomponents
+slug: "/supercomponents"
+description: "`SuperComponent` lets you wrap a complete pipeline and use it like a single component. This is helpful when you want to simplify the interface of a complex pipeline, reuse it in different contexts, or expose only the necessary inputs and outputs."
+---
+
+# SuperComponents
+
+`SuperComponent` lets you wrap a complete pipeline and use it like a single component. This is helpful when you want to simplify the interface of a complex pipeline, reuse it in different contexts, or expose only the necessary inputs and outputs.
+
+## `@super_component` decorator (recommended)
+
+Haystack now provides a simple `@super_component` decorator for wrapping a pipeline as a component. All you need is to create a class with the decorator, and to include an `pipeline` attribute.
+
+With this decorator, the `to_dict` and `from_dict` serialization is optional, as is the input and output mapping.
+
+### Example
+
+The custom HybridRetriever example SuperComponent below turns your query into embeddings, then runs both a BM25 search and an embedding-based search at the same time. It finally merges those two result sets and returns the combined documents.
+
+The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples:
+
+```shell
+pip install sentence-transformers-haystack
+```
+
+```python
+# pip install haystack-ai datasets sentence-transformers-haystack
+
+from haystack import Document, Pipeline, super_component
+from haystack.components.joiners import DocumentJoiner
+from haystack_integrations.components.embedders.sentence_transformers import (
+    SentenceTransformersTextEmbedder,
+)
+from haystack.components.retrievers import (
+    InMemoryBM25Retriever,
+    InMemoryEmbeddingRetriever,
+)
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+from datasets import load_dataset
+
+
+@super_component
+class HybridRetriever:
+    def __init__(
+        self,
+        document_store: InMemoryDocumentStore,
+        embedder_model: str = "BAAI/bge-small-en-v1.5",
+    ):
+        embedding_retriever = InMemoryEmbeddingRetriever(document_store)
+        bm25_retriever = InMemoryBM25Retriever(document_store)
+        text_embedder = SentenceTransformersTextEmbedder(embedder_model)
+        document_joiner = DocumentJoiner()
+
+        self.pipeline = Pipeline()
+        self.pipeline.add_component("text_embedder", text_embedder)
+        self.pipeline.add_component("embedding_retriever", embedding_retriever)
+        self.pipeline.add_component("bm25_retriever", bm25_retriever)
+        self.pipeline.add_component("document_joiner", document_joiner)
+
+        self.pipeline.connect("text_embedder", "embedding_retriever")
+        self.pipeline.connect("bm25_retriever", "document_joiner")
+        self.pipeline.connect("embedding_retriever", "document_joiner")
+
+
+dataset = load_dataset("HaystackBot/medrag-pubmed-chunk-with-embeddings", split="train")
+docs = [
+    Document(content=doc["contents"], embedding=doc["embedding"]) for doc in dataset
+]
+document_store = InMemoryDocumentStore()
+document_store.write_documents(docs)
+
+query = "What treatments are available for chronic bronchitis?"
+
+result = HybridRetriever(document_store).run(text=query, query=query)
+print(result)
+```
+
+### Input Mapping
+
+You can optionally map the input names of your SuperComponent to the actual sockets inside the pipeline.
+
+```python
+input_mapping = {"query": ["retriever.query", "prompt.query"]}
+```
+
+### Output Mapping
+
+You can also map the pipeline's output sockets that you want to expose to the SuperComponent's output names.
+
+```python
+output_mapping = {"llm.replies": "replies"}
+```
+
+If you don’t provide mappings, SuperComponent will try to auto-detect them. So, if multiple components have outputs with the same name, we recommend using `output_mapping` to avoid conflicts.
+
+## SuperComponent class
+
+Haystack also gives you an option to inherit from SuperComponent class. This option requires `to_dict` and `from_dict` serialization, as well as the input and output mapping described above.
+
+### Example
+
+Here is a simple example of initializing a `SuperComponent` with a pipeline:
+
+```python
+from haystack import Pipeline, SuperComponent
+
+with open("pipeline.yaml", "r") as file:
+    pipeline = Pipeline.load(file)
+
+super_component = SuperComponent(pipeline)
+```
+
+The example pipeline below retrieves relevant documents based on a user query, builds a custom prompt using those documents, then sends the prompt to an `OpenAIChatGenerator` to create an answer. The `SuperComponent` wraps the pipeline so it can be run with a simple input (`query`) and returns a clean output (`replies`).
+
+```python
+from haystack import Pipeline, SuperComponent
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.retrievers import InMemoryBM25Retriever
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.dataclasses import Document
+
+document_store = InMemoryDocumentStore()
+documents = [
+    Document(content="Paris is the capital of France."),
+    Document(content="London is the capital of England."),
+]
+document_store.write_documents(documents)
+
+prompt_template = [
+    ChatMessage.from_user(
+    '''
+    According to the following documents:
+    {% for document in documents %}
+    {{document.content}}
+    {% endfor %}
+    Answer the given question: {{query}}
+    Answer:
+    '''
+    )
+]
+
+prompt_builder = ChatPromptBuilder(template=prompt_template, required_variables="*")
+
+pipeline = Pipeline()
+pipeline.add_component("retriever", InMemoryBM25Retriever(document_store=document_store))
+pipeline.add_component("prompt_builder", prompt_builder)
+pipeline.add_component("llm", OpenAIChatGenerator())
+pipeline.connect("retriever.documents", "prompt_builder.documents")
+pipeline.connect("prompt_builder.prompt", "llm.messages")
+
+# Create a super component with simplified input/output mapping
+wrapper = SuperComponent(
+    pipeline=pipeline,
+    input_mapping={
+        "query": ["retriever.query", "prompt_builder.query"],
+    },
+    output_mapping={
+        "llm.replies": "replies",
+        "retriever.documents": "documents"
+    }
+)
+
+# Run the pipeline with simplified interface
+result = wrapper.run(query="What is the capital of France?")
+print(result)
+{'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>,
+ _content=[TextContent(text='The capital of France is Paris.')],...)
+```
+
+## Type Checking and Static Code Analysis
+
+Creating SuperComponents using the @super_component decorator can induce type or linting errors. One way to avoid these issues is to add the exposed public methods to your SuperComponent. Here's an example:
+
+```python
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+
+    def run(self, *, documents: list[Document]) -> dict[str, list[Document]]: ...
+    def warm_up(self) -> None:  # noqa: D102
+        ...
+```
+
+## Ready-Made SuperComponents
+
+You can see two implementations of SuperComponents already integrated in Haystack:
+
+- [DocumentPreprocessor](../../pipeline-components/preprocessors/documentpreprocessor.mdx)
+- [MultiFileConverter](../../pipeline-components/converters/multifileconverter.mdx)
+- [OpenSearchHybridRetriever](../../pipeline-components/retrievers/opensearchhybridretriever.mdx)

docs-website/docs/concepts/concepts-overview.mdx

@@ -0,0 +1,56 @@
+---
+title: "Haystack Concepts Overview"
+id: concepts-overview
+slug: "/concepts-overview"
+description: "Haystack provides all the tools you need to build custom agents and RAG pipelines with LLMs that work for you. This includes everything from prototyping to deployment. This page discusses the most important concepts Haystack operates on."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+
+# Haystack Concepts Overview
+
+Haystack provides all the tools you need to build custom agents and RAG pipelines with LLMs that work for you. This includes everything from prototyping to deployment. This page discusses the most important concepts Haystack operates on.
+
+### Components
+
+Haystack offers various components, each performing different kinds of tasks. You can see the whole variety in the **PIPELINE COMPONENTS** section in the left-side navigation. These are often powered by the latest Large Language Models (LLMs) and transformer models. Code-wise, they are Python classes with methods you can directly call. Most commonly, all you need to do is initialize the component with the required parameters and then run it with a `run()` method.
+
+Working on this level with Haystack components is a hands-on approach. Components define the name and the type of all of their inputs and outputs. The Component API reduces complexity and makes it easier to [create custom components](components/custom-components.mdx), for example, for third-party APIs and databases. Haystack validates the connections between components before running the pipeline and, if needed, generates error messages with instructions on fixing the errors.
+
+#### Generators
+
+[Generators](../pipeline-components/generators.mdx) are responsible for generating text responses after you give them a prompt. They are specific for each LLM technology (OpenAI, Cohere, local models, and others). There are two types of Generators: chat and non-chat:
+
+- The chat ones enable chat completion and are designed for conversational contexts. It expects a list of messages to interact with the user.
+- The non-chat Generators use LLMs for simpler text generation (for example, translating or summarizing text).
+
+Read more about various Generators in our [guides](../pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx).
+
+#### Retrievers
+
+[Retrievers](../pipeline-components/retrievers.mdx) go through all the documents in a Document Store, select the ones that match the user query, and pass it on to the next component. There are various Retrievers that are customized for specific Document Stores. This means that they can handle specific requirements for each database using customized parameters.
+
+For example, for Elasticsearch Document Store, you will find both the Document Store and Retriever packages in its GitHub [repo](https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch).
+
+### Document Stores
+
+[Document Store](document-store.mdx) is an object that stores your documents in Haystack, like an interface to a storage database. It uses specific functions like `write_documents()` or `delete_documents()` to work with data. Various components have access to the Document Store and can interact with it by, for example, reading or writing Documents.
+
+If you are working with more complex pipelines in Haystack, you can use a [`DocumentWriter`](../pipeline-components/writers/documentwriter.mdx) component to write data into Document Stores for you
+
+### Data Classes
+
+You can use different [data classes](data-classes.mdx) in Haystack to carry the data through the system. The data classes are mostly likely to appear as inputs or outputs of your pipelines.
+
+`Document` class contains information to be carried through the pipeline. It can be text, metadata, tables, or binary data. Documents can be written into Document Stores but also written and read by other components.
+
+`Answer` class holds not only the answer generated in a pipeline but also the originating query and metadata.
+
+### Pipelines
+
+Finally, you can combine various components, Document Stores, and integrations into [pipelines](pipelines.mdx) to create powerful and customizable systems. It is a highly flexible system that allows you to have simultaneous flows, standalone components, loops, and other types of connections. You can have the preprocessing, indexing, and querying steps all in one pipeline, or you can split them up according to your needs.
+
+If you want to reuse pipelines, you can save them into a convenient format (YAML, TOML, and more) on a disk or share them around using the [serialization](pipelines/serialization.mdx) process.
+
+Here is a short Haystack pipeline, illustrated:
+<ClickableImage src="/img/00f5fe8-Pipeline_Illustrations_2.png" alt="RAG architecture overview showing query flow through retrieval and generation stages, with document stores providing context for the language model" />

docs-website/docs/concepts/data-classes.mdx

@@ -0,0 +1,315 @@
+---
+title: "Data Classes"
+id: data-classes
+slug: "/data-classes"
+description: "In Haystack, there are a handful of core classes that are regularly used in many different places. These are classes that carry data through the system and you are likely to interact with these as either the input or output of your pipeline."
+---
+
+# Data Classes
+
+In Haystack, there are a handful of core classes that are regularly used in many different places. These are classes that carry data through the system and you are likely to interact with these as either the input or output of your pipeline.
+
+Haystack uses data classes to help components communicate with each other in a simple and modular way. By doing this, data flows seamlessly through the Haystack pipelines. This page goes over the available data classes in Haystack: ByteStream, Answer (along with its variants ExtractedAnswer and GeneratedAnswer), ChatMessage, FileContent, ImageContent, Document, and StreamingChunk, explaining how they contribute to the Haystack ecosystem.
+
+You can check out the detailed parameters in our [Data Classes](/reference/data-classes-api) API reference.
+
+### Answer
+
+#### Overview
+
+The `Answer` class serves as the base for responses generated within Haystack, containing the answer's data, the originating query, and additional metadata.
+
+#### Key Features
+
+- Adaptable data handling, accommodating any data type (`data`).
+- Query tracking for contextual relevance (`query`).
+- Extensive metadata support for detailed answer description.
+
+#### Attributes
+
+```python
+@dataclass
+class Answer:
+    data: Any
+    query: str
+    meta: Dict[str, Any]
+```
+
+### ExtractedAnswer
+
+#### Overview
+
+`ExtractedAnswer` is a subclass of `Answer` that deals explicitly with answers derived from Documents, offering more detailed attributes.
+
+#### Key Features
+
+- Includes reference to the originating `Document`.
+- Score attribute to quantify the answer's confidence level.
+- Optional start and end indices for pinpointing answer location within the source.
+
+#### Attributes
+
+```python
+@dataclass
+class ExtractedAnswer:
+    query: str
+    score: float
+    data: Optional[str] = None
+    document: Optional[Document] = None
+    context: Optional[str] = None
+    document_offset: Optional["Span"] = None
+    context_offset: Optional["Span"] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+```
+
+### GeneratedAnswer
+
+#### Overview
+
+`GeneratedAnswer` extends the `Answer` class to accommodate answers generated from multiple Documents.
+
+#### Key Features
+
+- Handles string-type data.
+- Links to a list of `Document` objects, enhancing answer traceability.
+
+#### Attributes
+
+```python
+@dataclass
+class GeneratedAnswer:
+    data: str
+    query: str
+    documents: List[Document]
+    meta: Dict[str, Any] = field(default_factory=dict)
+```
+
+### ByteStream
+
+#### Overview
+
+`ByteStream` represents binary object abstraction in the Haystack framework and is crucial for handling various binary data formats.
+
+#### Key Features
+
+- Holds binary data and associated metadata.
+- Optional MIME type specification for flexibility.
+- File interaction methods (`to_file`, `from_file_path`, `from_string`) for easy data manipulation.
+
+#### Attributes
+
+```python
+@dataclass(repr=False)
+class ByteStream:
+    data: bytes
+    meta: Dict[str, Any] = field(default_factory=dict, hash=False)
+    mime_type: Optional[str] = field(default=None)
+```
+
+#### Example
+
+```python
+from haystack.dataclasses.byte_stream import ByteStream
+
+image = ByteStream.from_file_path("dog.jpg")
+```
+
+### ChatMessage
+
+`ChatMessage` is the central abstraction to represent a message for a LLM. It contains role, metadata and several types of content, including text, tool calls and tool calls results.
+
+Read the detailed documentation for the `ChatMessage` data class on a dedicated [ChatMessage](data-classes/chatmessage.mdx) page.
+
+### FileContent
+
+`FileContent` represents a file payload that can be attached to a `ChatMessage`, including base64 data, MIME type, filename, and provider-specific metadata.
+
+Read the detailed documentation for the `FileContent` data class on a dedicated [FileContent](data-classes/filecontent.mdx) page.
+
+### ImageContent
+
+`ImageContent` represents image-based content used in multimodal chat messages and vision-language pipelines.
+
+Read the detailed documentation for the `ImageContent` data class on a dedicated [ImageContent](data-classes/imagecontent.mdx) page.
+
+### Document
+
+#### Overview
+
+`Document` represents a central data abstraction in Haystack, capable of holding text, tables, and binary data.
+
+#### Key Features
+
+- Unique ID for each document.
+- Multiple content types are supported: text, binary (`blob`).
+- Custom metadata and scoring for advanced document management.
+- Optional embedding for AI-based applications.
+
+#### Attributes
+
+```python
+@dataclass
+class Document(metaclass=_BackwardCompatible):
+    id: str = field(default="")
+    content: Optional[str] = field(default=None)
+    blob: Optional[ByteStream] = field(default=None)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    score: Optional[float] = field(default=None)
+    embedding: Optional[List[float]] = field(default=None)
+    sparse_embedding: Optional[SparseEmbedding] = field(default=None)
+```
+
+#### Example
+
+```python
+from haystack import Document
+
+documents = Document(
+    content="Here are the contents of your document",
+    embedding=[0.1] * 768,
+)
+```
+
+### StreamingChunk
+
+#### Overview
+
+`StreamingChunk` represents a partially streamed LLM response, enabling real-time LLM response processing. It encapsulates a segment of streamed content along with associated metadata and provides comprehensive information about the streaming state.
+
+#### Key Features
+
+- String-based content representation for text chunks
+- Support for tool calls and tool call results
+- Component tracking and metadata management
+- Streaming state indicators (start, finish reason)
+- Content block indexing for multi-part responses
+
+#### Attributes
+
+```python
+@dataclass
+class StreamingChunk:
+    content: str
+    meta: dict[str, Any] = field(default_factory=dict, hash=False)
+    component_info: Optional[ComponentInfo] = field(default=None)
+    index: Optional[int] = field(default=None)
+    tool_calls: Optional[list[ToolCallDelta]] = field(default=None)
+    tool_call_result: Optional[ToolCallResult] = field(default=None)
+    start: bool = field(default=False)
+    finish_reason: Optional[FinishReason] = field(default=None)
+    reasoning: Optional[ReasoningContent] = field(default=None)
+```
+
+#### Example
+
+```python
+from haystack.dataclasses import StreamingChunk, ToolCallDelta, ReasoningContent
+
+# Basic text chunk
+chunk = StreamingChunk(
+    content="Hello world",
+    start=True,
+    meta={"model": "gpt-5-mini"},
+)
+
+# Tool call chunk
+tool_chunk = StreamingChunk(
+    content="",
+    tool_calls=[
+        ToolCallDelta(
+            index=0,
+            tool_name="calculator",
+            arguments='{"operation": "add", "a": 2, "b": 3}',
+        ),
+    ],
+    index=0,
+    start=False,
+    finish_reason="tool_calls",
+)
+
+# Reasoning chunk
+reasoning_chunk = StreamingChunk(
+    content="",
+    reasoning=ReasoningContent(
+        reasoning_text="Thinking step by step about the answer.",
+    ),
+    index=0,
+    start=True,
+    meta={"model": "gpt-4.1-mini"},
+)
+```
+
+### ToolCallDelta
+
+#### Overview
+
+`ToolCallDelta` represents a tool call prepared by the model, usually contained in an assistant message during streaming.
+
+#### Attributes
+
+```python
+@dataclass
+class ToolCallDelta:
+    index: int
+    tool_name: Optional[str] = field(default=None)
+    arguments: Optional[str] = field(default=None)
+    id: Optional[str] = field(default=None)
+    extra: Optional[Dict[str, Any]] = field(default=None)
+```
+
+### ComponentInfo
+
+#### Overview
+
+The `ComponentInfo` class represents information about a component within a Haystack pipeline. It is used to track the type and name of components that generate or process data, aiding in debugging, tracing, and metadata management throughout the pipeline.
+
+#### Key Features
+
+- Stores the type of the component (including module and class name).
+- Optionally stores the name assigned to the component in the pipeline.
+- Provides a convenient class method to create a `ComponentInfo` instance from a `Component` object.
+
+#### Attributes
+
+```python
+@dataclass
+class ComponentInfo:
+    type: str
+    name: Optional[str] = field(default=None)
+
+    @classmethod
+    def from_component(cls, component: Component) -> "ComponentInfo": ...
+```
+
+#### Example
+
+```python
+from haystack.dataclasses.streaming_chunk import ComponentInfo
+from haystack.core.component import Component
+
+
+class MyComponent(Component): ...
+
+
+component = MyComponent()
+info = ComponentInfo.from_component(component)
+print(info.type)  # e.g., 'my_module.MyComponent'
+print(info.name)  # Name assigned in the pipeline, if any
+```
+
+### SparseEmbedding
+
+#### Overview
+
+The `SparseEmbedding` class represents a sparse embedding: a vector where most values are zeros.
+
+#### Attributes
+
+- `indices`: List of indices of non-zero elements in the embedding.
+- `values`: List of values of non-zero elements in the embedding.
+
+### Tool
+
+`Tool` is a data class representing a tool that Language Models can prepare a call for.
+
+Read the detailed documentation for the `Tool` data class on a dedicated [Tool](../tools/tool.mdx) page.

docs-website/docs/concepts/data-classes/chatmessage.mdx

@@ -0,0 +1,413 @@
+---
+title: "ChatMessage"
+id: chatmessage
+slug: "/chatmessage"
+description: "`ChatMessage` is the central abstraction to represent a message for a LLM. It contains role, metadata and several types of content, including text, images, tool calls, tool call results, and reasoning content."
+---
+
+# ChatMessage
+
+`ChatMessage` is the central abstraction to represent a message for a LLM. It contains role, metadata and several types of content, including text, images, tool calls, tool call results, and reasoning content.
+
+To create a `ChatMessage` instance, use `from_user`, `from_system`, `from_assistant`, and `from_tool` class methods.
+
+The [content](#types-of-content) of the `ChatMessage` can then be inspected using the `text`, `texts`, `image`, `images`, `file`, `files`, `tool_call`, `tool_calls`, `tool_call_result`, `tool_call_results`, `reasoning`, and `reasonings` properties.
+
+If you are looking for the details of this data class methods and parameters, head over to our [API documentation](/reference/data-classes-api#chatmessage).
+
+## Types of Content
+
+`ChatMessage` currently supports `TextContent`, `ImageContent`, `FileContent`, `ToolCall`, `ToolCallResult`, and `ReasoningContent` types of content:
+
+```python
+@dataclass
+class TextContent:
+    """
+    The textual content of a chat message.
+
+    :param text: The text content of the message.
+    """
+
+    text: str
+
+
+@dataclass
+class ToolCall:
+    """
+    Represents a Tool call prepared by the model, usually contained in an assistant message.
+
+    :param tool_name: The name of the Tool to call.
+    :param arguments: The arguments to call the Tool with.
+    :param id: The ID of the Tool call.
+    :param extra: Dictionary of extra information about the Tool call. Use to store provider-specific
+        information. To avoid serialization issues, values should be JSON serializable.
+    """
+
+    tool_name: str
+    arguments: Dict[str, Any]
+    id: Optional[str] = None  # noqa: A003
+    extra: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class ToolCallResult:
+    """
+    Represents the result of a Tool invocation.
+
+    :param result: The result of the Tool invocation.
+    :param origin: The Tool call that produced this result.
+    :param error: Whether the Tool invocation resulted in an error.
+    """
+
+    result: str | Sequence[TextContent | ImageContent]
+    origin: ToolCall
+    error: bool
+
+
+@dataclass
+class ImageContent:
+    """
+    The image content of a chat message.
+
+    :param base64_image: A base64 string representing the image.
+    :param mime_type: The MIME type of the image (e.g. "image/png", "image/jpeg").
+        Providing this value is recommended, as most LLM providers require it.
+        If not provided, the MIME type is guessed from the base64 string, which can be slow and not always reliable.
+    :param detail: Optional detail level of the image (only supported by OpenAI). One of "auto", "high", or "low".
+    :param meta: Optional metadata for the image.
+    :param validation: If True (default), a validation process is performed:
+        - Check whether the base64 string is valid;
+        - Guess the MIME type if not provided;
+        - Check if the MIME type is a valid image MIME type.
+        Set to False to skip validation and speed up initialization.
+    """
+
+    base64_image: str
+    mime_type: Optional[str] = None
+    detail: Optional[Literal["auto", "high", "low"]] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+    validation: bool = True
+
+
+@dataclass
+class FileContent:
+    """
+    The file content of a chat message.
+
+    :param base64_data: A base64 string representing the file.
+    :param mime_type: The MIME type of the file (e.g. "application/pdf").
+        Providing this value is recommended, as most LLM providers require it.
+        If not provided, the MIME type is guessed from the base64 string, which can be slow and not always reliable.
+    :param filename: Optional filename of the file. Some LLM providers use this information.
+    :param extra: Dictionary of extra information about the file. Can be used to store provider-specific information.
+        To avoid serialization issues, values should be JSON serializable.
+    :param validation: If True (default), a validation process is performed:
+        - Check whether the base64 string is valid;
+        - Guess the MIME type if not provided.
+        Set to False to skip validation and speed up initialization.
+    """
+
+    base64_data: str
+    mime_type: str | None = None
+    filename: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+    validation: bool = True
+
+
+@dataclass
+class ReasoningContent:
+    """
+    Represents the optional reasoning content prepared by the model, usually contained in an assistant message.
+
+    :param reasoning_text: The reasoning text produced by the model.
+    :param extra: Dictionary of extra information about the reasoning content. Use to store provider-specific
+        information. To avoid serialization issues, values should be JSON serializable.
+    """
+
+    reasoning_text: str
+    extra: Dict[str, Any] = field(default_factory=dict)
+```
+
+The `ImageContent` and `FileContent` dataclasses also provide two convenience class methods: `from_file_path` and `from_url`.
+For more details, refer to our [API documentation](/reference/data-classes-api).
+
+## Working with a ChatMessage
+
+The following examples demonstrate how to create a `ChatMessage` and inspect its properties.
+
+### from_user with TextContent
+
+```python
+from haystack.dataclasses import ChatMessage
+
+user_message = ChatMessage.from_user("What is the capital of Australia?")
+
+print(user_message)
+>>> ChatMessage(
+>>>    _role=<ChatRole.USER: 'user'>,
+>>>    _content=[TextContent(text='What is the capital of Australia?')],
+>>>    _name=None,
+>>>    _meta={}
+>>>)
+
+print(user_message.text)
+>>> What is the capital of Australia?
+
+print(user_message.texts)
+>>> ['What is the capital of Australia?']
+```
+
+### from_user with TextContent and ImageContent
+
+```python
+from haystack.dataclasses import ChatMessage, ImageContent
+
+lion_image_url = (
+    "https://images.unsplash.com/photo-1546182990-dffeafbe841d?"
+	"ixlib=rb-4.0&q=80&w=1080&fit=max"
+)
+
+image_content = ImageContent.from_url(lion_image_url, detail="low")
+
+user_message = ChatMessage.from_user(
+	content_parts=[
+		"What does the image show?",
+		image_content
+		])
+
+print(user_message)
+>>> ChatMessage(
+>>>     _role=<ChatRole.USER: 'user'>,
+>>>     _content=[
+>>>         TextContent(text='What does the image show?'),
+>>>         ImageContent(
+>>>             base64_image='/9j/4...',
+>>>             mime_type='image/jpeg',
+>>>             detail='low',
+>>>             meta={
+>>>                 'content_type': 'image/jpeg',
+>>>                 'url': '...'
+>>>             }
+>>>         )
+>>>     ],
+>>>     _name=None,
+>>>     _meta={}
+>>> )
+
+print(user_message.text)
+>>> What does the image show?
+
+print(user_message.texts)
+>>> ['What does the image show?']
+
+print(user_message.image)
+>>> ImageContent(
+>>>     base64_image='/9j/4...',
+>>>     mime_type='image/jpeg',
+>>>     detail='low',
+>>>     meta={
+>>>         'content_type': 'image/jpeg',
+>>>         'url': '...'
+>>>     }
+>>> )
+```
+
+### from_user with TextContent and FileContent
+
+```python
+from haystack.dataclasses import ChatMessage, FileContent
+
+paper_url = "https://arxiv.org/pdf/2309.08632"
+
+file_content = FileContent.from_url(paper_url)
+
+user_message = ChatMessage.from_user(
+	content_parts=[
+        file_content,
+        "Summarize this paper in 100 words."
+		])
+
+print(user_message)
+>>> ChatMessage(
+>>>     _role=<ChatRole.USER: 'user'>,
+>>>     _content=[
+>>>         FileContent(
+>>>             base64_data='JVBERi0...',
+>>>             mime_type='application/pdf',
+>>>             filename='2309.08632',
+>>>             extra={}
+>>>         ),
+>>>         TextContent(text='Summarize this paper in 100 words.')
+>>>     ],
+>>>     _name=None,
+>>>     _meta={}
+>>> )
+
+print(user_message.text)
+>>> Summarize this paper in 100 words.
+
+print(user_message.texts)
+>>> ['Summarize this paper in 100 words.']
+
+print(user_message.file)
+>>> FileContent(
+>>>     base64_data='JVBERi0...',
+>>>     mime_type='application/pdf',
+>>>     filename='2309.08632',
+>>>     extra={}
+>>> )
+```
+
+### from_assistant with TextContent
+
+```python
+from haystack.dataclasses import ChatMessage
+
+assistant_message = ChatMessage.from_assistant("How can I assist you today?")
+
+print(assistant_message)
+>>> ChatMessage(
+>>>    _role=<ChatRole.ASSISTANT: 'assistant'>,
+>>>    _content=[TextContent(text='How can I assist you today?')],
+>>>    _name=None,
+>>>    _meta={}
+>>>)
+
+print(assistant_message.text)
+>>> How can I assist you today?
+
+print(assistant_message.texts)
+>>> ['How can I assist you today?']
+```
+
+### from_assistant with ToolCall
+
+```python
+from haystack.dataclasses import ChatMessage, ToolCall
+
+tool_call = ToolCall(tool_name="weather_tool", arguments={"location": "Rome"})
+
+assistant_message_w_tool_call = ChatMessage.from_assistant(tool_calls=[tool_call])
+
+print(assistant_message_w_tool_call)
+>>> ChatMessage(
+>>>    _role=<ChatRole.ASSISTANT: 'assistant'>,
+>>>    _content=[ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None)],
+>>>    _name=None,
+>>>    _meta={}
+>>>)
+
+print(assistant_message_w_tool_call.text)
+>>> None
+
+print(assistant_message_w_tool_call.texts)
+>>> []
+
+print(assistant_message_w_tool_call.tool_call)
+>>> ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None)
+
+print(assistant_message_w_tool_call.tool_calls)
+>>> [ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None)]
+
+print(assistant_message_w_tool_call.tool_call_result)
+>>> None
+
+print(assistant_message_w_tool_call.tool_call_results)
+>>> []
+```
+
+### from_tool
+
+```python
+from haystack.dataclasses import ChatMessage
+
+tool_message = ChatMessage.from_tool(tool_result="temperature: 25°C", origin=tool_call, error=False)
+
+print(tool_message)
+>>> ChatMessage(
+>>>    _role=<ChatRole.TOOL: 'tool'>,
+>>>    _content=[ToolCallResult(
+>>>							   result='temperature: 25°C',
+>>>                origin=ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None),
+>>>                error=False
+>>>                )],
+>>>    _name=None,
+>>>    _meta={}
+>>>)
+
+print(tool_message.text)
+>>> None
+
+print(tool_message.texts)
+>>> []
+
+print(tool_message.tool_call)
+>>> None
+
+print(tool_message.tool_calls)
+>>> []
+
+print(tool_message.tool_call_result)
+>>> ToolCallResult(
+>>>     result='temperature: 25°C',
+>>>     origin=ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None),
+>>>     error=False
+>>> )
+
+print(tool_message.tool_call_results)
+>>> [
+>>>     ToolCallResult(
+>>>         result='temperature: 25°C',
+>>>         origin=ToolCall(tool_name='weather_tool', arguments={'location': 'Rome'}, id=None),
+>>>         error=False
+>>>     )
+>>> ]
+```
+
+## Migrating from Legacy ChatMessage (before v2.9)
+
+In Haystack 2.9, we updated the `ChatMessage` data class for greater flexibility and support for multiple content types: text, tool calls, and tool call results.
+
+There are some breaking changes involved, so we recommend reviewing this guide to migrate smoothly.
+
+### Creating a ChatMessage
+
+You can no longer directly initialize `ChatMessage` using `role`, `content`, and `meta`.
+
+- Use the following class methods instead: `from_assistant`, `from_user`, `from_system`, and `from_tool`.
+- Replace the `content` parameter with `text`.
+
+```python
+from haystack.dataclasses import ChatMessage
+
+# LEGACY - DOES NOT WORK IN 2.9.0
+message = ChatMessage(role=ChatRole.USER, content="Hello!")
+
+# Use the class method instead
+message = ChatMessage.from_user("Hello!")
+```
+
+### Accessing ChatMessage Attributes
+
+- The legacy `content` attribute is now internal (`_content`).
+- Inspect `ChatMessage` attributes using the following properties:
+  - `role`
+  - `meta`
+  - `name`
+  - `text` and `texts`
+  - `image` and `images`
+  - `tool_call` and `tool_calls`
+  - `tool_call_result` and `tool_call_results`
+  - `reasoning` and `reasonings`
+
+```python
+from haystack.dataclasses import ChatMessage
+
+message = ChatMessage.from_user("Hello!")
+
+# LEGACY - DOES NOT WORK IN 2.9.0
+print(message.content)
+
+# Use the appropriate property instead
+print(message.text)
+```

docs-website/docs/concepts/data-classes/filecontent.mdx

@@ -0,0 +1,119 @@
+---
+title: "FileContent"
+id: filecontent
+slug: "/filecontent"
+description: "`FileContent` represents file payloads in chat messages, including base64 data, MIME type, filename, and provider-specific metadata."
+---
+
+# FileContent
+
+`FileContent` represents a file payload that can be attached to a [`ChatMessage`](chatmessage.mdx). Use it when a chat model accepts file inputs, such as PDFs or other documents, together with the user's text prompt.
+
+If you need the full list of parameters and methods, see the [`FileContent` API reference](/reference/data-classes-api#filecontent).
+
+## Attributes
+
+```python
+@dataclass
+class FileContent:
+    base64_data: str
+    mime_type: str | None = None
+    filename: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+    validation: bool = True
+```
+
+- `base64_data` stores the file content as a base64-encoded string.
+- `mime_type` identifies the file type, for example `application/pdf`. Providing it explicitly is recommended because many model providers require it.
+- `filename` is optional, but some providers use it when processing uploaded files.
+- `extra` can store provider-specific metadata. Values should be JSON serializable.
+- `validation` checks that `base64_data` is valid and tries to infer the MIME type when one is not provided.
+
+## Create from a file path
+
+Use `from_file_path` to read a local file, base64-encode it, infer the MIME type from the path, and populate the filename.
+
+```python
+from haystack.dataclasses import ChatMessage, FileContent
+
+file_content = FileContent.from_file_path("data/attention-is-all-you-need.pdf")
+
+message = ChatMessage.from_user(
+    content_parts=[
+        file_content,
+        "Summarize the key ideas in this paper.",
+    ]
+)
+```
+
+Pass `filename` or `extra` when a provider expects a specific filename or provider-specific options:
+
+```python
+file_content = FileContent.from_file_path(
+    "data/report.pdf",
+    filename="quarterly-report.pdf",
+    extra={"source": "finance"},
+)
+```
+
+## Create from a URL
+
+Use `from_url` to download a file and convert it into a `FileContent` instance.
+
+```python
+from haystack.dataclasses import FileContent
+
+file_content = FileContent.from_url(
+    "https://example.com/reports/quarterly-report.pdf",
+    timeout=30,
+)
+```
+
+If no filename is provided, Haystack uses the final path segment of the URL.
+
+## Create from base64 data
+
+If you already have file bytes, encode them and pass the MIME type explicitly.
+
+```python
+import base64
+from pathlib import Path
+
+from haystack.dataclasses import FileContent
+
+data = Path("data/manual.pdf").read_bytes()
+file_content = FileContent(
+    base64_data=base64.b64encode(data).decode("utf-8"),
+    mime_type="application/pdf",
+    filename="manual.pdf",
+)
+```
+
+Set `validation=False` only when the base64 data and MIME type are already trusted and you want to skip validation.
+
+## Inspect files in a ChatMessage
+
+After adding `FileContent` to a `ChatMessage`, use the `file` and `files` properties to access file payloads.
+
+```python
+from haystack.dataclasses import ChatMessage, FileContent
+
+file_content = FileContent.from_file_path("data/invoice.pdf")
+message = ChatMessage.from_user(content_parts=[file_content, "Extract the invoice total."])
+
+print(message.file)
+print(message.files)
+```
+
+`message.file` returns the first file payload, or `None` if there are no files. `message.files` returns all file payloads.
+
+## Serialization
+
+Use `to_dict` and `from_dict` to serialize and restore file content.
+
+```python
+payload = file_content.to_dict()
+restored = FileContent.from_dict(payload)
+```
+
+For tracing, Haystack replaces the full base64 payload with a placeholder so large files are not sent to the tracing backend.

docs-website/docs/concepts/data-classes/imagecontent.mdx

@@ -0,0 +1,247 @@
+---
+title: "ImageContent"
+id: imagecontent
+slug: "/imagecontent"
+description: "`ImageContent` represents image-based content in Haystack chat messages and multimodal pipelines."
+---
+
+# ImageContent
+
+`ImageContent` is a Haystack data class used to represent image-based content in chat messages and multimodal AI pipelines.
+
+It is commonly used with:
+
+* multimodal LLMs
+* vision-language models
+* image-aware chat applications
+* document/image processing workflows
+
+`ImageContent` stores images as base64-encoded strings together with metadata such as MIME type and image detail level.
+
+If you are looking for the full API reference, see the [API documentation](/reference/data-classes-api#imagecontent).
+
+---
+
+# Creating ImageContent
+
+You can create an `ImageContent` object directly from a base64 string:
+
+```python
+from haystack.dataclasses import ImageContent
+
+image = ImageContent(base64_image="your_base64_encoded_image", mime_type="image/png")
+
+print(image)
+```
+
+---
+
+# Loading Images from a File Path
+
+The `from_file_path()` class method provides a convenient way to load local image files.
+
+```python
+from haystack.dataclasses import ImageContent
+
+image = ImageContent.from_file_path("sample.png", detail="low")
+
+print(image)
+```
+
+The optional `detail` parameter is currently supported by OpenAI vision models and accepts:
+
+* `"auto"`
+* `"high"`
+* `"low"`
+
+You can also resize images while loading:
+
+```python
+image = ImageContent.from_file_path("sample.png", size=(512, 512))
+```
+
+This helps reduce:
+
+* memory usage
+* processing time
+* payload size
+
+when working with multimodal LLM APIs.
+
+---
+
+# Loading Images from a URL
+
+You can also create an `ImageContent` object directly from an image URL:
+
+```python
+from haystack.dataclasses import ImageContent
+
+image = ImageContent.from_url(
+    "https://images.unsplash.com/photo-1546182990-dffeafbe841d",
+    detail="low",
+)
+
+print(image)
+```
+
+Internally, Haystack downloads the image and converts it into a base64 representation.
+
+---
+
+# Producing ImageContent with Converters
+
+In a pipeline, you usually don't create `ImageContent` objects by hand. Instead, you use converter components that read files and produce `ImageContent` for you:
+
+* [`ImageFileToImageContent`](../../pipeline-components/converters/imagefiletoimagecontent.mdx) converts local image files (such as PNG or JPEG) into `ImageContent` objects.
+* [`PDFToImageContent`](../../pipeline-components/converters/pdftoimagecontent.mdx) renders the pages of PDF files into `ImageContent` objects.
+
+```python
+from haystack.components.converters.image import (
+    ImageFileToImageContent,
+    PDFToImageContent,
+)
+
+image_converter = ImageFileToImageContent()
+image_contents = image_converter.run(sources=["image.jpg", "another_image.png"])[
+    "image_contents"
+]
+
+pdf_converter = PDFToImageContent()
+pdf_image_contents = pdf_converter.run(sources=["file.pdf"])["image_contents"]
+```
+
+Both converters accept the optional `detail` and `size` parameters, which are forwarded to the `ImageContent` objects they create.
+
+---
+
+# Using ImageContent with ChatMessage
+
+`ImageContent` is commonly used together with [`ChatMessage`](chatmessage.mdx) for multimodal conversations.
+
+```python
+from haystack.dataclasses import ChatMessage, ImageContent
+
+image = ImageContent.from_url(
+    "https://images.unsplash.com/photo-1546182990-dffeafbe841d",
+    detail="low",
+)
+
+message = ChatMessage.from_user(content_parts=["What does this image show?", image])
+
+print(message)
+```
+
+This allows multimodal LLMs to process both:
+
+* textual prompts
+* image inputs
+
+within the same message.
+
+For more dynamic prompts, you can build multimodal messages with [`ChatPromptBuilder`](../../pipeline-components/builders/chatpromptbuilder.mdx) using Jinja2 string templates. The `| templatize_part` filter inserts an `ImageContent` object as a structured content part instead of plain text:
+
+```python
+from haystack.components.builders import ChatPromptBuilder
+from haystack.dataclasses import ChatMessage, ImageContent
+
+template = """
+{% message role="user" %}
+Hello! I am {{user_name}}. What's the difference between the following images?
+{% for image in images %}
+{{ image | templatize_part }}
+{% endfor %}
+{% endmessage %}
+"""
+
+builder = ChatPromptBuilder(template=template)
+images = [
+    ImageContent.from_file_path("apple.jpg"),
+    ImageContent.from_file_path("kiwi.jpg"),
+]
+result = builder.run(user_name="John", images=images)
+
+print(result["prompt"])
+```
+
+---
+
+# Metadata
+
+The optional `meta` parameter allows you to attach custom metadata to the image.
+
+```python
+image = ImageContent.from_url(
+    "https://images.unsplash.com/photo-1546182990-dffeafbe841d",
+    meta={"source": "example-dataset"},
+)
+```
+
+This can be useful for:
+
+* tracing
+* dataset tracking
+* workflow metadata
+* custom application logic
+
+---
+
+# Validation
+
+By default, `ImageContent` validates:
+
+* base64 encoding
+* MIME type correctness
+* image MIME compatibility
+
+Validation can be disabled to improve performance:
+
+```python
+image = ImageContent(
+    base64_image="your_base64_encoded_image",
+    mime_type="image/png",
+    validation=False,
+)
+```
+
+---
+
+# Serialization
+
+`ImageContent` supports dictionary serialization.
+
+```python
+image_dict = image.to_dict()
+
+restored_image = ImageContent.from_dict(image_dict)
+```
+
+---
+
+# Displaying Images
+
+The `show()` method can display images directly in:
+
+* Jupyter notebooks
+* local desktop environments
+
+```python
+image.show()
+```
+
+This requires the `Pillow` package:
+
+```bash
+pip install pillow
+```
+
+---
+
+# Related Components
+
+`ImageContent` is frequently used with:
+
+* [`ChatMessage`](chatmessage.mdx) — to build multimodal messages
+* [`ChatPromptBuilder`](../../pipeline-components/builders/chatpromptbuilder.mdx) — to template multimodal prompts
+* [`ImageFileToImageContent`](../../pipeline-components/converters/imagefiletoimagecontent.mdx) — to convert image files into `ImageContent`
+* [`PDFToImageContent`](../../pipeline-components/converters/pdftoimagecontent.mdx) — to convert PDF pages into `ImageContent`

docs-website/docs/concepts/device-management.mdx

@@ -0,0 +1,145 @@
+---
+title: "Device Management"
+id: device-management
+slug: "/device-management"
+description: "This page discusses the concept of device management in the context of Haystack."
+---
+
+# Device Management
+
+This page discusses the concept of device management in the context of Haystack.
+
+Many Haystack components, such as `TransformersChatGenerator`, `AzureOpenAIChatGenerator`, and others, allow users the ability to pick and choose which language model is to be queried and executed. For components that interface with cloud-based services, the service provider automatically takes care of the details of provisioning the requisite hardware (like GPUs). However, if you wish to use models on your local machine, you’ll need to figure out how to deploy them on your hardware. Further complicating things, different ML libraries have different APIs to launch models on specific devices.
+
+To make the process of running inference on local models as straightforward as possible, Haystack uses a framework-agnostic device management implementation. Exposing devices through this interface means you no longer need to worry about library-specific invocations and device representations.
+
+## Concepts
+
+Haystack’s device management is built on the following abstractions:
+
+- `DeviceType`  - An enumeration that lists all the different types of supported devices.
+- `Device`  - A generic representation of a device composed of a `DeviceType` and a unique identifier. Together, it represents a single device in the group of all available devices.
+- `DeviceMap` - A mapping of strings to `Device` instances. The strings represent model-specific identifiers, usually model parameters. This allows us to map specific parts of a model to specific devices.
+- `ComponentDevice` - A tagged union of a single `Device` or a `DeviceMap` instance. Components that support local inference will expose an optional `device` parameter of this type in their constructor.
+
+With the above abstractions, Haystack can fully address any supported device that’s part of your local machine and can support the usage of multiple devices at the same time. Every component that supports local inference will internally handle the conversion of these generic representations to their backend-specific representations.
+
+:::info[Source Code]
+
+Find the full code for the abstractions above in the Haystack GitHub [repo](https://github.com/deepset-ai/haystack/blob/6a776e672fb69cc4ee42df9039066200f1baf24e/haystack/utils/device.py).
+:::
+
+## Usage
+
+:::info
+The examples below use the [`TransformersChatGenerator`](../pipeline-components/generators/transformerschatgenerator.mdx), which is part of the `transformers-haystack` integration. Install it with:
+
+```bash
+pip install transformers-haystack
+```
+:::
+
+To use a single device for inference, use either the `ComponentDevice.from_single` or `ComponentDevice.from_str` class method:
+
+```python
+from haystack.utils import ComponentDevice, Device
+from haystack_integrations.components.generators.transformers import (
+    TransformersChatGenerator,
+)
+
+device = ComponentDevice.from_single(Device.gpu(id=1))
+# Alternatively, use a PyTorch device string
+device = ComponentDevice.from_str("cuda:1")
+generator = TransformersChatGenerator(model="Qwen/Qwen3-0.6B", device=device)
+```
+
+To use multiple devices, use the `ComponentDevice.from_multiple` class method:
+
+```python
+from haystack.utils import ComponentDevice, Device, DeviceMap
+from haystack_integrations.components.generators.transformers import (
+    TransformersChatGenerator,
+)
+
+device_map = DeviceMap(
+    {
+        "encoder.layer1": Device.gpu(id=0),
+        "decoder.layer2": Device.gpu(id=1),
+        "self_attention": Device.disk(),
+        "lm_head": Device.cpu(),
+    },
+)
+device = ComponentDevice.from_multiple(device_map)
+generator = TransformersChatGenerator(model="Qwen/Qwen3-0.6B", device=device)
+```
+
+### Integrating Devices in Custom Components
+
+Components should expose an optional `device` parameter of type `ComponentDevice`.  Once exposed, they can determine what to do with it:
+
+- If `device=None`, the component can pass that to the backend. In this case, the backend decides which device the model will be placed on.
+- Alternatively, the component can attempt to automatically pick an available device before passing it to the backend using the `ComponentDevice.resolve_device` class method.
+
+Once the device has been resolved, the component can use the `ComponentDevice.to_*` methods to get the backend-specific representation of the underlying device, which is then passed to the backend.
+
+The `ComponentDevice` instance should be serialized in the component’s `to_dict` and `from_dict` methods.
+
+```python
+from haystack.utils import ComponentDevice, Device, DeviceMap
+
+class MyComponent(Component):
+    def __init__(self, device: Optional[ComponentDevice] = None):
+        # If device is None, automatically select a device.
+        self.device = ComponentDevice.resolve_device(device)
+
+    def warm_up(self):
+        # Call the framework-specific conversion method.
+        self.model = AutoModel.from_pretrained(
+            "deepset/bert-base-cased-squad2", device=self.device.to_hf()
+            )
+
+    def to_dict(self):
+        # Serialize the policy like any other (custom) data.
+        return default_to_dict(
+            self, device=self.device.to_dict() if self.device else None, ...
+        )
+
+    @classmethod
+    def from_dict(cls, data):
+        # Deserialize the device data inplace before passing
+        # it to the generic from_dict function.
+	    init_params = data["init_parameters"]
+        init_params["device"] = ComponentDevice.from_dict(init_params["device"])
+        return default_from_dict(cls, data)
+
+# Automatically selects a device.
+c = MyComponent(device=None)
+
+# Uses the first GPU available.
+c = MyComponent(device=ComponentDevice.from_str("cuda:0"))
+
+# Uses the CPU.
+c = MyComponent(device=ComponentDevice.from_single(Device.cpu()))
+
+# Allow the component to use multiple devices using a device map.
+c = MyComponent(device=ComponentDevice.from_multiple(DeviceMap({
+    "layer1": Device.cpu(),
+    "layer2": Device.gpu(1),
+    "layer3": Device.disk()
+})))
+```
+
+If the component’s backend provides a more specialized API to manage devices, it could add an additional init parameter that acts as a conduit. For instance, `TransformersChatGenerator` exposes a `huggingface_pipeline_kwargs` parameter through which Hugging Face-specific `device_map`  arguments can be passed:
+
+```python
+from haystack_integrations.components.generators.transformers import (
+    TransformersChatGenerator,
+)
+
+generator = TransformersChatGenerator(
+    model="Qwen/Qwen3-0.6B",
+    huggingface_pipeline_kwargs={"device_map": "balanced"},
+)
+```
+
+In such cases, ensure that the parameter precedence and selection behavior is clearly documented. In the case of `TransformersChatGenerator`, the device map passed through the `huggingface_pipeline_kwargs` parameter overrides the explicit `device` parameter and is documented as such.

docs-website/docs/concepts/document-store.mdx

@@ -0,0 +1,103 @@
+---
+title: "Document Store"
+id: document-store
+slug: "/document-store"
+description: "You can think of the Document Store as a database that stores your data and provides them to the Retriever at query time. Learn how to use Document Store in a pipeline or how to create your own."
+---
+
+# Document Store
+
+You can think of the Document Store as a database that stores your data and provides them to the Retriever at query time. Learn how to use Document Store in a pipeline or how to create your own.
+
+Document Store is an object that stores your documents. In Haystack, a Document Store is different from a component, as it doesn't have the `run()` method. You can think of it as an interface to your database – you put the information there, or you can look through it. This means that a Document Store is not a piece of a pipeline but rather a tool that the components of a pipeline have access to and can interact with.
+
+:::tip[Work with Retrievers]
+
+The most common way to use a Document Store in Haystack is to fetch documents using a Retriever. A Document Store will often have a corresponding Retriever to get the most out of specific technologies. See more information in our [Retriever](../pipeline-components/retrievers.mdx) documentation.
+:::
+
+:::note[How to choose a Document Store?]
+
+To learn about different types of Document Stores and their strengths and disadvantages, head to the [Choosing a Document Store](document-store/choosing-a-document-store.mdx) page.
+:::
+
+### DocumentStore Protocol
+
+Document Stores in Haystack are designed to use the following methods as part of their protocol:
+
+- `count_documents` returns the number of documents stored in the given store as an integer.
+- `filter_documents` returns a list of documents that match the provided filters.
+- `write_documents` writes or overwrites documents into the given store and returns the number of documents that were written as an integer.
+- `delete_documents` deletes all documents with given `document_ids` from the Document Store.
+
+### Initialization
+
+To use a Document Store in a pipeline, you must initialize it first.
+
+See the installation and initialization details for each Document Store in the "Document Stores" section in the navigation panel on your left.
+
+### Work with Documents
+
+Convert your data into `Document` objects before writing them into a Document Store along with its metadata and document ID.
+
+The ID field is mandatory, so if you don’t choose a specific ID yourself, Haystack will do its best to come up with a unique ID based on the document’s information and assign it automatically. However, since Haystack uses the document’s contents to create an ID, two identical documents might have identical IDs. Keep it in mind as you update your documents, as the ID will not be updated automatically.
+
+```python
+document_store = ChromaDocumentStore()
+documents = [
+    Document(
+        meta={"name": DOCUMENT_NAME, ...}, id="document_unique_id", content="this is content"
+    ),
+  	...
+]
+document_store.write_documents(documents)
+```
+
+To write documents into the `InMemoryDocumentStore`, simply call the `.write_documents()` function:
+
+```python
+document_store.write_documents(
+    [
+        Document(content="My name is Jean and I live in Paris."),
+        Document(content="My name is Mark and I live in Berlin."),
+        Document(content="My name is Giorgio and I live in Rome."),
+    ],
+)
+```
+
+:::note[`DocumentWriter`]
+
+See `DocumentWriter` component [docs](../pipeline-components/writers/documentwriter.mdx) to write your documents into a Document Store in a pipeline.
+:::
+
+### DuplicatePolicy
+
+The `DuplicatePolicy` is a class that defines the different options for handling documents with the same ID in a `DocumentStore`. It has three possible values:
+
+- **OVERWRITE**: Indicates that if a document with the same ID already exists in the `DocumentStore`, it should be overwritten with the new document.
+- **SKIP**: If a document with the same ID already exists, the new document will be skipped and not added to the `DocumentStore`.
+- **FAIL**: Raises an error if a document with the same ID already exists in the `DocumentStore`. It prevents duplicate documents from being added.
+
+Here is an example of how you could apply the policy to skip the existing document:
+
+```python
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.types import DuplicatePolicy
+
+document_store = InMemoryDocumentStore()
+document_writer = DocumentWriter(
+    document_store=document_store,
+    policy=DuplicatePolicy.SKIP,
+)
+```
+
+### Custom Document Store
+
+All custom document stores must implement the [protocol](https://github.com/deepset-ai/haystack/blob/13804293b1bb79743e5a30e980b76a0561dcfaf8/haystack/document_stores/types/protocol.py) with four mandatory methods: `count_documents`,`filter_documents`, `write_documents`, and `delete_documents`.
+
+The `init` function should indicate all the specifics for the chosen database or vector store.
+
+We also recommend having a custom corresponding Retriever to get the most out of a specific Document Store.
+
+See [Creating Custom Document Stores](document-store/creating-custom-document-stores.mdx) page for more details.

docs-website/docs/concepts/document-store/choosing-a-document-store.mdx

@@ -0,0 +1,183 @@
+---
+title: "Choosing a Document Store"
+id: choosing-a-document-store
+slug: "/choosing-a-document-store"
+description: "This article goes through different types of Document Stores and explains their advantages and disadvantages."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+
+# Choosing a Document Store
+
+Whether you are developing a chatbot, a RAG system, or an image captioner, at some point, it's likely for your AI
+application to compare the input it gets with the information it already knows.
+
+Haystack currently has integrations with seven categories of Document Stores:
+
+- **Vector Databases** — purpose-built for embedding search and semantic retrieval
+- **Search Engines** — full-text search engines extended with vector (kNN) capabilities
+- **Relational Databases** — SQL databases with vector search via plugins or extensions
+- **Document / NoSQL Databases** — flexible document stores with vector search added on top
+- **In-memory Key-Value Stores** — ultra-low-latency stores with HNSW vector search
+- **Vector Index Libraries** — lightweight in-process vector similarity search, no external service
+- **Multi-model Databases** — single engine supporting graph, document, and vector data models
+
+Here is an overview of all the integrations currently available, grouped by category:
+
+<ClickableImage src="/img/document-stores-overview.svg" alt="Overview of DocumentStore integrations in Haystack grouped by category: Vector Databases, Search Engines, Relational Databases, Document/NoSQL Databases, In-memory Key-Value Stores, Vector Index Libraries, Multi-model Databases" className="img-light-bg img-full-width" />
+
+## DocumentStore Integrations Available in Haystack
+
+Haystack integrations come in two tiers. **Core integrations** are built and maintained by the Haystack team — they are tested against every release, follow the same API conventions, and come with full documentation and support. **External integrations** are contributed and maintained by the community; they extend Haystack's reach but are not covered by the core release cycle.
+
+The tables below list every available integration alongside the key properties you need to choose the right one for your use case.
+
+#### Core integrations
+
+| Integration | Category | Engine Type | Open Source | Async Support | Retrievers |
+| --- | --- | --- | --- | --- | --- |
+| ArcadeDB | Multi-model Database | Multi-model database (graph, document, key-value) with HNSW vector search via HTTP/JSON API | Yes | No | Embedding |
+| AlloyDB | Relational Database | Managed PostgreSQL-compatible database (Google Cloud) with pgvector extension | No | Yes | Embedding, Keyword |
+| ArangoDB | Multi-model Database | Multi-model database (graph, document, key-value) with AQL vector search (requires v3.12+) | Yes (BUSL) | No | Embedding |
+| Astra | Document / NoSQL Database | Cloud-native managed NoSQL (Apache Cassandra-based) with vector search via DataStax JSON API | No | No | Embedding |
+| Azure AI Search | Search Engine | Managed cloud search service (Microsoft Azure AI Search) with HNSW vector search | No | No | BM25, Embedding, Hybrid |
+| Chroma | Vector Database | Purpose-built vector database | Yes | Yes | Embedding |
+| Elasticsearch | Search Engine | Distributed search & analytics engine with BM25 + vector (kNN) search | Partial | Yes | BM25, Embedding, SQL |
+| FAISS | Vector Index Library | In-memory vector similarity search library (Meta/Facebook) with JSON file for metadata | Yes | No | Embedding |
+| FalkorDB | Graph Database | OpenCypher graph database with ANN vector search | Yes (SSPL) | No | Embedding, Cypher |
+| MongoDB Atlas | Document / NoSQL Database | Cloud document database with Atlas Vector Search and full-text search | No | Yes | Embedding, Full-text |
+| Oracle | Relational Database | Oracle with native AI Vector Search, HNSW vector index and DBMS_SEARCH full-text keyword index | No | Yes | Embedding, Keyword |
+| OpenSearch | Search Engine | Distributed search engine (AWS fork of Elasticsearch) with BM25 + kNN vector search | Yes | Yes | BM25, Embedding, Hybrid, Metadata, SQL |
+| PGVector | Relational Database | Relational database (PostgreSQL) with the `pgvector` extension for vector similarity search | Yes | Yes | Embedding, Keyword |
+| Pinecone | Vector Database | Managed cloud vector database | No | Yes | Embedding |
+| Qdrant | Vector Database | Purpose-built vector database with dense + sparse embedding support | Yes | Yes | Embedding, Sparse Embedding, Hybrid |
+| Supabase | Relational Database | Managed cloud Supabase — a wrapper over PgvectorDocumentStore with Supabase-specific defaults | Yes | Yes | Embedding, Keyword |
+| Valkey | In-memory Key-Value Store | In-memory key-value store (Redis fork) with HNSW vector search via `glide` client | Yes | Yes | Embedding |
+| Vespa | Search Engine | Distributed search & serving engine with BM25 lexical + HNSW vector (ANN) search | Yes | No | BM25, Embedding |
+| Weaviate | Vector Database | Purpose-built vector database with hybrid search support | Yes | Yes | BM25, Embedding, Hybrid |
+
+#### External integrations
+
+| Integration | Category | Engine Type | Open Source | Async Support | Retrievers |
+| --- | --- | --- | --- | --- | --- |
+| Couchbase | Document / NoSQL Database | Distributed NoSQL document database with vector search via Search Service | Partial | Yes | Embedding, Full-text |
+| LanceDB | Vector Database | Embedded vector database built on the Lance columnar format, optimized for multimodal data | Yes | Yes | Embedding, Full-text, Hybrid |
+| Milvus | Vector Database | Open-source vector database built for scalable similarity search | Yes | No | Embedding |
+| Needle | Search Engine | Managed RAG-as-a-service platform with built-in document storage and vector search | No | Yes | Embedding, Sparse Embedding, Hybrid |
+| Neo4j | Multi-model Database | Graph database with native vector index support for combined graph traversal and similarity search | Partial | No | Embedding |
+| SingleStore | Relational Database | Distributed SQL database with native vector search and full-text search support | No | Yes | Embedding, Full-text, Keyword |
+
+## Vector Databases
+
+- Purpose-built for vector and embedding search
+- Advanced indexing techniques for efficient similarity search
+- Designed for high scalability and availability with large volumes of high-dimensional data
+- Most support metadata filtering alongside vector search
+- Increasingly adding hybrid (vector + keyword) search support
+- Mostly open source, widely available as managed cloud services
+
+**Best for** semantic search over large document corpora — e.g. a knowledge base where users search by meaning rather than exact keywords.
+
+- [Chroma](../../document-stores/chromadocumentstore.mdx)
+- [Pinecone](../../document-stores/pinecone-document-store.mdx)
+- [Qdrant](../../document-stores/qdrant-document-store.mdx)
+- [Weaviate](../../document-stores/weaviatedocumentstore.mdx)
+- [LanceDB](https://haystack.deepset.ai/integrations/lancedb) (external integration)
+- [Milvus](https://haystack.deepset.ai/integrations/milvus-document-store) (external integration)
+
+## Search Engines
+
+- Originally built for full-text (BM25) search, with vector (kNN) capabilities added later
+- Excellent support for text data, tokenisation, and language-aware querying
+- Scale both horizontally and vertically in production environments
+- Strong foundation for hybrid search combining keyword and semantic retrieval
+- Battle-tested in enterprise environments with mature tooling and observability
+
+**Best for** enterprise search or log analytics where both full-text (BM25) and vector search are needed — e.g. an e-commerce product search with filters.
+
+- Azure AI Search ([AzureAISearchDocumentStore](../../document-stores/azureaisearchdocumentstore.mdx))
+- [Elasticsearch](../../document-stores/elasticsearch-document-store.mdx)
+- [OpenSearch](../../document-stores/opensearch-document-store.mdx)
+- [Needle](https://haystack.deepset.ai/integrations/needle) (external integration)
+- [Vespa](../../document-stores/vespadocumentstore.mdx)
+
+## Relational Databases
+
+- Standard SQL databases extended with vector search via plugins or extensions
+- Vectors live alongside relational data, enabling combined vector + SQL queries in a single store
+- Lower operational overhead when PostgreSQL is already part of the stack
+- Vector search performance is lower than purpose-built databases, but sufficient for many use cases
+- Familiar tooling, transactions, and data integrity guarantees of a relational database
+
+**Best for** use cases where documents live alongside structured relational data — e.g. a product catalogue where vector search and SQL JOINs are both needed.
+
+- [AlloyDB](../../document-stores/alloydbdocumentstore.mdx)
+- [Oracle](../../document-stores/oracledocumentstore.mdx)
+- [PGVector](../../document-stores/pgvectordocumentstore.mdx)
+- [Supabase](../../document-stores/supabasedocumentstore.mdx)
+- [SingleStore](https://haystack.deepset.ai/integrations/singlestore) (external integration)
+
+## Document / NoSQL Databases
+
+- General-purpose document stores with vector search added on top
+- Flexible, schema-less data model suited for heterogeneous document collections
+- Horizontal scaling and high availability inherited from the underlying NoSQL engine
+- Good choice when the database is already in use and adding a separate vector store is undesirable
+- Vector search performance may trail behind purpose-built databases
+
+**Best for** applications already that want to add RAG capabilities without introducing a new infrastructure component.
+
+- Astra ([AstraDocumentStore](../../document-stores/astradocumentstore.mdx))
+- [MongoDB Atlas](../../document-stores/mongodbatlasdocumentstore.mdx)
+- [Couchbase](https://haystack.deepset.ai/integrations/couchbase-document-store) (external integration)
+
+## In-memory Key-Value Stores
+
+- In-memory architecture delivers extremely low read/write latency
+- Vector search (HNSW) layered on top of an existing caching infrastructure
+- Ideal when the stack already includes Valkey as a cache or session store
+- Data is ephemeral by default; persistence requires explicit configuration
+- Less suited for large corpora where memory cost becomes significant
+
+**Best for** low-latency, real-time retrieval — e.g. a chatbot that needs sub-millisecond response times.
+
+- [Valkey](../../document-stores/valkeydocumentstore.mdx)
+
+## Vector Index Libraries
+
+- Low-level, in-process vector similarity search — not a full database
+- No network overhead; runs entirely within the application process
+- Very efficient use of hardware resources (CPU/GPU)
+- Limited to vectors only; metadata must be managed separately (e.g. via a JSON file)
+- No built-in persistence, replication, or multi-client access
+
+**Best for** local prototyping, research, or small-scale applications where a lightweight in-process solution is preferred over running an external database server.
+
+- [FAISS](../../document-stores/faissdocumentstore.mdx)
+
+## Multi-model Databases
+
+- Single engine supporting multiple data models: graph, document, key-value, and vector
+- Eliminates the need to maintain separate databases for different data representations
+- Suited for knowledge graphs or applications with complex entity relationships
+- Vector search (HNSW) available alongside graph traversal and document queries
+- Smaller community and ecosystem compared to more established categories
+
+**Best for** applications requiring multiple data models in a single engine — e.g. a knowledge graph where entities are connected by relationships and also need vector similarity search.
+
+- [ArangoDB](../../document-stores/arangodocumentstore.mdx)
+- [ArcadeDB](../../document-stores/arcadedbdocumentstore.mdx)
+- [FalkorDB](../../document-stores/falkordbdocumentstore.mdx)
+- [Neo4j](https://haystack.deepset.ai/integrations/neo4j-document-store) (external integration)
+
+## The In-memory Document Store
+
+Haystack ships with an ephemeral document store that relies on pure Python data structures stored in memory, so it doesn't fall into any of the vector database categories above. This special Document Store is ideal for creating quick prototypes with small datasets. It doesn't require any special setup, and it can be used right away without installing additional dependencies.
+
+- [InMemoryDocumentStore](../../document-stores/inmemorydocumentstore.mdx)
+
+## Final Considerations
+
+It can be very challenging to pick one Document Store over another by only looking at pure performance, as even the slightest difference in the benchmark can produce a different leaderboard (for example, some benchmarks test the cloud services while others work on a reference machine). Thinking about including features like filtering or not can bring in a whole new set of complexities that make the comparison even harder.
+
+What's important for you to know is that the Document Store interface doesn't add much to the costs, and the relative performance of one vector database over another should stay the same when used within Haystack pipelines.

docs-website/docs/concepts/document-store/creating-custom-document-stores.mdx

@@ -0,0 +1,174 @@
+---
+title: "Creating Custom Document Stores"
+id: creating-custom-document-stores
+slug: "/creating-custom-document-stores"
+description: "Create your own Document Stores to manage your documents."
+---
+
+# Creating Custom Document Stores
+
+Create your own Document Stores to manage your documents.
+
+Custom Document Stores are resources that you can build and leverage in situations where a ready-made solution is not available in Haystack. For example:
+
+- You’re working with a vector store that’s not yet supported in Haystack.
+- You need a very specific retrieval strategy to search for your documents.
+- You want to customize the way Haystack reads and writes documents.
+
+Similar to [custom components](../components/custom-components.mdx), you can use a custom Document Store in a Haystack pipeline as long as you can import its code into your Python program. The best practice is distributing a custom Document Store as a standalone integration package.
+
+## Recommendations
+
+Before you start, there are a few recommendations we provide to ensure a custom Document Store behaves consistently with the rest of the Haystack ecosystem. At the end of the day, a Document Store is just Python code written in a way that Haystack can understand, but the way you name it, organize it, and distribute it can make a difference. None of these recommendations are mandatory, but we encourage you to follow as many as you can.
+
+### Naming Convention
+
+We recommend naming your Document Store following the format `<TECHNOLOGY>-haystack`, for example, `chroma-haystack`. This makes it consistent with the others, lowering the cognitive load for your users and easing discoverability.
+
+This naming convention applies to the name of the git repository (`https://github.com/your-org/example-haystack`) and the name of the Python package (`example-haystack`).
+
+### Structure
+
+More often than not, a Document Store can be fairly complex, and setting up a dedicated Git repository can be handy and future-proof. To ease this step, we prepared a [GitHub template](https://github.com/deepset-ai/custom-component) that provides the structure you need to host a custom Document Store in a dedicated repository. It includes the boilerplate for packaging, testing, and distributing your custom Document Store as a standalone Python package.
+
+See the instructions in the [template repository](https://github.com/deepset-ai/custom-component) to get started. You can also watch the [video walkthrough](https://www.youtube.com/watch?v=SWC0QecAMcI) for a step-by-step guide.
+
+### Packaging
+
+As with any other [Haystack integration](../integrations.mdx), a Document Store can be added to your Haystack applications by installing an additional Python package, for example, with `pip`. Once you have a Git repository hosting your Document Store and a `pyproject.toml` file to create an `example-haystack` package (using our [GitHub template](https://github.com/deepset-ai/custom-component)), it will be possible to `pip install` it directly from sources, for example:
+
+```shell
+pip install git+https://github.com/your-org/example-haystack.git
+```
+
+Though very practical to quickly deliver prototypes, if you want others to use your custom Document Store, we recommend you publish a package on PyPI so that it will be versioned and installable with simply:
+
+```shell
+pip install example-haystack
+```
+
+:::tip
+👍
+
+Our [GitHub template](https://github.com/deepset-ai/custom-component) ships a GitHub workflow that will automatically publish the Document Store package on PyPI.
+:::
+
+### Documentation
+
+We recommend thoroughly documenting your custom Document Store with a detailed README file and possibly generating API documentation using a static generator.
+
+For inspiration, see the [neo4j-haystack](https://github.com/prosto/neo4j-haystack) repository and its [documentation](https://prosto.github.io/neo4j-haystack/) pages.
+
+## Implementation
+
+### DocumentStore Protocol
+
+You can use any Python class as a Document Store, provided that it implements all the methods of the `DocumentStore` Python protocol defined in Haystack:
+
+```python
+class DocumentStore(Protocol):
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes this store to a dictionary.
+        """
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "DocumentStore":
+        """
+        Deserializes the store from a dictionary.
+        """
+
+    def count_documents(self) -> int:
+        """
+        Returns the number of documents stored.
+        """
+
+    def filter_documents(
+        self,
+        filters: Optional[Dict[str, Any]] = None,
+    ) -> List[Document]:
+        """
+        Returns the documents that match the filters provided.
+        """
+
+    def write_documents(
+        self,
+        documents: List[Document],
+        policy: DuplicatePolicy = DuplicatePolicy.FAIL,
+    ) -> int:
+        """
+        Writes (or overwrites) documents into the DocumentStore, return the number of documents that was written.
+        """
+
+    def delete_documents(self, document_ids: List[str]) -> None:
+        """
+        Deletes all documents with a matching document_ids from the DocumentStore.
+        """
+```
+
+The `DocumentStore` interface supports the basic CRUD operations you would normally perform on a database or a storage system, and mostly generic components like [`DocumentWriter`](../../pipeline-components/writers/documentwriter.mdx) use it.
+
+### Additional Methods
+
+Usually, a Document Store comes with additional methods that can provide advanced search functionalities. These methods are not part of the `DocumentStore` protocol and don’t follow any particular convention. We designed it like this to provide maximum flexibility to the Document Store when using any specific features of the underlying database.
+
+Some additional methods that are not part of the `DocumentStore` protocol, but are implemented by most Document Stores in Haystack, include:
+
+```python
+def delete_all_documents(recreate_index: bool = False)
+def update_by_filter(filters: dict[str, Any], meta: dict[str, Any], refresh: bool = False) -> int:
+def delete_by_filter(filters: dict[str, Any]) -> int:
+```
+These methods are not part of the Protocol but highly recommended to implement in your custom Document Store, as users often expect them to be available.
+
+For example, Haystack wouldn’t get in the way when your Document Store defines a specific `search` method that takes a long list of parameters that only make sense in the context of a particular vector database. Normally, a [Retriever](../../pipeline-components/retrievers.mdx) component would then use this additional search method.
+
+### Retrievers
+
+To get the most out of your custom Document Store, in most cases, you would need to create one or more accompanying Retrievers that use the additional search methods mentioned above. Before proceeding and implementing your custom Retriever, it might be helpful to learn more about  [Retrievers](../../pipeline-components/retrievers.mdx) in general through the Haystack documentation.
+
+From the implementation perspective, Retrievers in Haystack are like any other custom component. For more details, refer to the [creating custom components](../components/custom-components.mdx) documentation page.
+
+Although not mandatory, we encourage you to follow more specific [naming conventions](../../pipeline-components/retrievers.mdx#naming-conventions) for your custom Retriever.
+
+### Serialization
+
+Haystack requires every component to be representable by a Python dictionary for correct serialization implementation. Some components, such as Retrievers and Writers, maintain a reference to a Document Store instance. Therefore, `DocumentStore` classes should implement the `from_dict` and `to_dict` methods. This allows to rebuild an instance after reading a pipeline from a file.
+
+For a practical example of what to serialize in a custom Document Store, consider a database client you created using an IP address and a database name. When constructing the dictionary to return in `to_dict`, you would store the IP address and the database name, not the database client instance.
+
+### Secrets Management
+
+There's a likelihood that users will need to provide sensitive data, such as passwords, API keys, or private URLs, to create a Document Store instance. This sensitive data could potentially be leaked if it's passed around in plain text.
+
+Haystack has a specific way to wrap sensitive data into special objects called Secrets. This prevents the data from being leaked during serialization roundtrips. We strongly recommend using this feature extensively for data security (better safe than sorry!).
+
+You can read more about Secret Management in Haystack [documentation](../secret-management.mdx).
+
+### Testing
+
+Haystack comes with some testing functionalities you can use in a custom Document Store. In particular, an empty class inheriting from `DocumentStoreBaseTests` would already run the standard tests that any Document Store is expected to pass in order to work properly.
+
+### Implementation Tips
+
+- The best way to learn how to write a custom Document Store is to look at the existing ones: the `InMemoryDocumentStore`, which is part of Haystack, or the [`ElasticsearchDocumentStore`](https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch), which is a Core Integration, are good places to start.
+- When starting from scratch, it might be easier to create the four CRUD methods of the `DocumentStore` protocol one at a time and test them one at a time as well. For example:
+  1. Implement the logic for `count_documents`.
+  2. In your `test_document_store.py` module, define the test class `TestDocumentStore(CountDocumentsTest)`. Note how we only inherit from the specific testing mix-in `CountDocumentsTest`.
+  3. Make the tests pass.
+  4. Implement the logic for `write_documents`.
+  5. Change `test_document_store.py` so that your class now also derives from the `WriteDocumentsTest` mix-in: `TestDocumentStore(CountDocumentsTest, WriteDocumentsTest)`.
+  6. Keep iterating with the remaining methods.
+- Having a notebook where users can try out your Document Store in a full pipeline can really help adoption, and it’s a great source of documentation. Our [haystack-cookbook](https://github.com/deepset-ai/haystack-cookbook) repository has good visibility, and we encourage contributors to create a PR and add their own.
+
+Verifying that the implementation meets all `DocumentStoreBaseTests` [tests](https://github.com/deepset-ai/haystack/blob/main/haystack/testing/document_store.py) is the minimum requirement for a custom Document Store to be consistent with the rest of the Haystack ecosystem.
+
+But, ideally making it compatible with the ``DocumentStoreBaseExtendedTests`` tests is a good way to ensure that your Document Store meets all the common used functionalities that users expect from a Document Store, such as `delete_all_documents` or `update_by_filter`.
+
+If the technology you are using for your Document Store supports asynchronous operations, we recommend implementing `async` versions of the methods in the `DocumentStore` protocol as well. This allows users to take advantage of async features in their applications and pipelines, improving performance and scalability.
+
+## Get Featured on the Integrations Page
+
+The [Integrations web page](https://haystack.deepset.ai/integrations) makes Haystack integrations visible to the community, and it’s a great opportunity to showcase your work. Once your Document Store is usable and properly packaged, you can open a pull request in the [haystack-integrations](https://github.com/deepset-ai/haystack-integrations) GitHub repository to add an integration tile.
+
+See the [integrations documentation page](../integrations.mdx#how-do-i-showcase-my-integration) for more details.

docs-website/docs/concepts/experimental-package.mdx

@@ -0,0 +1,67 @@
+---
+title: "Experimental Package"
+id: experimental-package
+slug: "/experimental-package"
+description: "Try out new experimental features with Haystack."
+---
+
+# Experimental Package
+
+Try out new experimental features with Haystack.
+
+The `haystack-experimental` package allows you to test new experimental features without committing to their official release. Its main goal is to gather user feedback and iterate on new features quickly.
+
+Check out the `haystack-experimental` [GitHub repository](https://github.com/deepset-ai/haystack-experimental) for the latest catalog of available features, or take a look at our [Experiments API Reference](/reference).
+
+### Installation
+
+For simplicity, every release of `haystack-experimental` includes all the available experiments at that time. To install the latest features, run:
+
+```shell
+pip install -U haystack-experimental
+```
+
+:::info
+The latest version of the experimental package is only tested against the latest version of Haystack. Compatibility with older versions of Haystack is not guaranteed.
+:::
+
+### Lifecycle
+
+Each experimental feature has a default lifespan of 3 months starting from the date of the first non-pre-release build that includes it. Once it reaches the end of its lifespan, we will remove it from `haystack-experimental` and either:
+
+- Merge the feature into Haystack and publish it with the next minor release,
+- Release the feature as an integration, or
+- Drop the feature.
+
+### Usage
+
+You can import the experimental new features like any other Haystack integration package:
+
+```python
+from haystack.dataclasses import ChatMessage
+from haystack_experimental.components.generators import FoobarGenerator
+
+c = FoobarGenerator()
+c.run([ChatMessage.from_user("What's an experiment? Be brief.")])
+```
+
+Experiments can also override existing Haystack features. For example, you can opt into an experimental type of `Pipeline` by changing the usual import:
+
+```python
+# from haystack import Pipeline
+from haystack_experimental import Pipeline
+
+pipe = Pipeline()
+# ...
+pipe.run(...)
+```
+
+## Additional References
+
+🧑‍🍳 Cookbooks:
+
+- [Improving Retrieval with Auto-Merging and Hierarchical Document Retrieval](https://haystack.deepset.ai/cookbook/auto_merging_retriever)
+- [Invoking APIs with OpenAPITool](https://haystack.deepset.ai/cookbook/openapitool)
+- [Conversational RAG using Memory](https://haystack.deepset.ai/cookbook/conversational_rag_using_memory)
+- [Define & Run Tools](https://haystack.deepset.ai/cookbook/tools_support)
+- [Newsletter Sending Agent with Experimental Haystack Tools](https://haystack.deepset.ai/cookbook/newsletter-agent)

docs-website/docs/concepts/integrations.mdx

@@ -0,0 +1,60 @@
+---
+title: "Introduction to Integrations"
+id: integrations
+slug: "/integrations"
+description: "The Haystack ecosystem integrates with many other technologies, such as vector databases, model providers and even custom components made by the community.  Here you can explore our integrations, which may be maintined by deepset, or submitted by others."
+---
+
+# Introduction to Integrations
+
+The Haystack ecosystem integrates with many other technologies, such as vector databases, model providers and even custom components made by the community.  Here you can explore our integrations, which may be maintined by deepset, or submitted by others.
+
+Haystack integrates with a number of other technologies and tools. For example, you can use a number of different model providers or databases with Haystack. 
+
+There are two main types of integrations:
+
+- **Maintained by deepset:** All of the integrations we maintain are hosted in the [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) repository.
+- **Maintained by our partners or community:** These are integrations that you, our partners, or anyone else can build and maintain themselves. Given they comply with some of our requirements, we will also showcase these on our website.
+
+## What are integrations?
+
+An integration is any type of external technology that can be used to extend the capabilities of the Haystack framework. Some integration examples are those providing access to model providers like OpenAI or Cohere, to databases like Weaviate and Qdrant, or even to monitoring tools such as Traceloop. They can be components, Document Stores, or any other feature that can be used with Haystack.
+
+We maintain a list of available integrations on the [Haystack Integrations](https://haystack.deepset.ai/integrations) page, where you can see which integrations we maintain or which have been contributed by the community.
+
+An integrations page focuses on explaining how Haystack integrates with that technology. For example, the OpenAI integration page will provide a summary of the various ways Haystack and OpenAI can work together.
+
+Here are the integration types you can currently choose from:
+
+- **Model Provider**: You can see how we integrate with different model providers and the available components through these integrations
+- **Document Store**: These are the databases and vector stores you can use with your Haystack pipelines.
+- **Evaluation Framework**: Evaluation frameworks that are supported by Haystack that you can use to evaluate Haystack pipelines.
+- **Monitoring Tool**: These are tools like Chainlit and Traceloop that integrate with Haystack and provide monitoring and observability capabilities.
+- **Data Ingestion**: These are the integrations that allow you to ingest and use data from different resources, such as Notion, Mastodon, and others.
+- **Custom Component**: Some integrations that cover very unique use cases are often contributed and maintained by our community members. We list these integrations under the _Custom Component_ tag.
+
+## How do I use an integration?
+
+Each page dedicated to an integration contains installation instructions and basic usage instructions. For example, the OpenAI integration page gives you an overview of the different ways in which you can interact with OpenAI.
+
+## How can I create an integration?
+
+The most common types of integrations are custom components and Document Stores. Integrations such as model providers might even include multiple custom components. Have a look at these documentation pages that will guide you through the requirements for each integration type:
+
+- [Creating Custom Components](components/custom-components.mdx)
+- [Creating Custom Document Stores](document-store/creating-custom-document-stores.mdx)
+
+Check out the [video walkthrough](https://www.youtube.com/watch?v=SWC0QecAMcI) for a step-by-step guide on how to use the [custom-component template](https://github.com/deepset-ai/custom-component) to create a Haystack integration.
+
+## How do I showcase my integration?
+
+To make your integration visible to the Haystack community, contribute it to our [haystack-integrations](https://github.com/deepset-ai/haystack-integrations) GitHub repository. There are several requirements you have to follow:
+
+- Make sure your contribution is [packaged](https://packaging.python.org/en/latest/), installable, and runnable. We suggest using [hatch](https://hatch.pypa.io/latest/) for this purpose.
+- Provide the GitHub repo and issue link.
+- Create a Pull Request in the [haystack-integrations](https://github.com/deepset-ai/haystack-integrations) repo by following the [draft-integration.md](https://github.com/deepset-ai/haystack-integrations/blob/main/draft-integration.md) and include a clear explanation of what your integration is. This page should include:
+  - Installation instructions
+  - A list of the components the integration includes
+  - Examples of how to use it with clear/runnable code
+  - Licensing information
+  - (Optionally) Documentation and/or API docs that you’ve generated for your repository

docs-website/docs/concepts/jinja-templates.mdx

@@ -0,0 +1,58 @@
+---
+title: "Jinja Templates"
+id: jinja-templates
+slug: "/jinja-templates"
+description: "Learn how Jinja templates work with Haystack components."
+---
+
+# Jinja Templates
+
+Learn how Jinja templates work with Haystack components.
+
+Jinja templates are text structures that contain placeholders for generating dynamic content. These placeholders are filled in when the template is rendered. You can check out the full list of Jinja2 features in the [original documentation](https://jinja.palletsprojects.com/en/3.0.x/templates/).
+
+You can use these templates in Haystack [Builders](../pipeline-components/builders.mdx), [OutputAdapter](../pipeline-components/converters/outputadapter.mdx), and [ConditionalRouter](../pipeline-components/routers/conditionalrouter.mdx) components.
+
+Here is an example of `OutputAdapter` using a short Jinja template to output only the content field of the first document in the arrays of documents:
+
+```python
+from haystack import Document
+from haystack.components.converters import OutputAdapter
+
+adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+input_data = {"documents": [Document(content="Test content")]}
+expected_output = {"output": "Test content"}
+assert adapter.run(**input_data) == expected_output
+```
+
+### Using Python f‑strings with Jinja
+
+When you embed Jinja placeholders inside a Python f‑string, you must escape Jinja’s `{` and `}` by doubling them (so `{{ var }}` becomes `{{{{ var }}}}`). Otherwise, Python will consume the braces and the Jinja variable won’t be found.
+
+Preferred template:
+
+```python
+template = """
+Language: {{ language }}
+Question: {{ question }}
+"""
+# pass both variables when rendering
+```
+
+It you need to use an f‑string (escape braces):
+
+```python
+language = "en"
+template = f"""
+Language: {language}
+Question: {{{{ question }}}}
+"""
+```
+
+## Safety Features
+
+Due to how we use Jinja in some Components, there are some security considerations to take into account. Jinja works by executing embedded in templates, so it’s _imperative_ that they stem from a trusted source. If the template is allowed to be customized by the end user, it can potentially lead to remote code execution.
+
+To mitigate this risk, Jinja templates are executed and rendered in a [sandbox environment](https://jinja.palletsprojects.com/en/3.1.x/sandbox/). While this approach is safer, it's also less flexible and limits the expressiveness of the template. If you need the more advanced functionality of Jinja templates, components that use them provide an `unsafe` init parameter - setting it to `False` will disable the sandbox environment and enable unsafe template rendering.
+
+With unsafe template rendering, the [OutputAdapter](../pipeline-components/converters/outputadapter.mdx) and [ConditionalRouter](../pipeline-components/routers/conditionalrouter.mdx) components allow their `output_type` to be set to one of the [Haystack data classes](data-classes.mdx) such as `ChatMessage`, `Document`, or `Answer`.

docs-website/docs/concepts/metadata-filtering.mdx

@@ -0,0 +1,155 @@
+---
+title: "Metadata Filtering"
+id: metadata-filtering
+slug: "/metadata-filtering"
+description: "This page provides a detailed explanation of how to apply metadata filters at query time."
+---
+
+# Metadata Filtering
+
+This page provides a detailed explanation of how to apply metadata filters at query time.
+
+When you index documents into your Document Store, you can attach metadata to them. One example is the `DocumentLanguageClassifier`, which adds the language of the document's content to its metadata. Components like `MetadataRouter` can then route documents based on their metadata.
+
+You can then use the metadata to filter your search queries, allowing you to narrow down the results by focusing on specific criteria. This ensures your Retriever fetches answers from the most relevant subset of your data.
+
+To illustrate how metadata filters work, imagine you have a set of annual reports from various companies. You may want to perform a search on just a specific year and just on a small selection of companies. This can reduce the workload of the Retriever and also ensure that you get more relevant results.
+
+## Filtering Types
+
+Filters are defined as a dictionary or nested dictionaries that can be of two types: Comparison or Logic.
+
+### Comparison
+
+Comparison operators help search your metadata fields according the specified conditions.
+
+Comparison dictionaries must contain the following keys:
+
+\-`field`: the name of one of the meta fields of a document, such as `meta.years`.
+
+\-`operator`: must be one of the following:
+
+```
+    - `==`
+    - `!=`
+    - `>`
+    - `>=`
+    - `<`
+    - `<=`
+    - `in`
+    - `not in`
+```
+
+:::info
+The available comparison operators may vary depending on the specific Document Store integration. For example, the `ChromaDocumentStore` supports two additional operators: `contains` and `not contains`. Find the details about the supported filters in the specific integration’s API reference.
+:::
+
+\-`value`: takes a single value or (in the case of "in" and “not in”) a list of values.
+
+#### Example
+
+Here is an example of a simple filter in the form of a dictionary. The filter selects documents classified as “article” in the `type` meta field of the document:
+
+```python
+filters = {"field": "meta.type", "operator": "==", "value": "article"}
+```
+
+### Logic
+
+Logical operators can be used to create a nested dictionary, allowing you to apply multiple `fields` as filter conditions. Logic dictionaries must contain the following keys:
+
+\-`operator`: usually one of the following:
+
+```
+    - `NOT`
+    - `OR`
+    - `AND`
+```
+
+:::info
+The available logic operators may vary depending on the specific Document Store integration. For example, the `ChromaDocumentStore` doesn’t support the `NOT` operator. Find the details about the supported filters in the specific integration’s API reference.
+:::
+
+\-`conditions`: must be a list of dictionaries, either of type Comparison or Logic.
+
+#### Nested Filter Example
+
+Here is a more complex filter that uses both Comparison and Logic to find documents where:
+
+- Meta field `type` is "article",
+- Meta field `date` is between 1420066800 and 1609455600 (a specific date range),
+- Meta field `rating` is greater than or equal to 3,
+- Documents are either classified as `genre`  ["economy", "politics"] `OR` the meta field `publisher` is "nytimes".
+
+```python
+filters = {
+    "operator": "AND",
+    "conditions": [
+        {"field": "meta.type", "operator": "==", "value": "article"},
+        {"field": "meta.date", "operator": ">=", "value": 1420066800},
+        {"field": "meta.date", "operator": "<", "value": 1609455600},
+        {"field": "meta.rating", "operator": ">=", "value": 3},
+        {
+            "operator": "OR",
+            "conditions": [
+                {
+                    "field": "meta.genre",
+                    "operator": "in",
+                    "value": ["economy", "politics"],
+                },
+                {"field": "meta.publisher", "operator": "==", "value": "nytimes"},
+            ],
+        },
+    ],
+}
+```
+
+## Filters Usage
+
+Filters can be applied either through the `Retriever` class or directly within Document Stores.
+
+In the `Retriever` class, filters are passed through the `filters` argument. When working with a pipeline, filters can be provided to `Pipeline.run()`, which will automatically route them to the `Retriever` class (refer to the [pipelines documentation](pipelines.mdx) for more information on working with pipelines).
+
+The example below shows how filters can be passed to Retrievers within a pipeline:
+
+```python
+pipeline.run(
+    data={
+        "retriever": {
+            "query": "Why did the revenue increase?",
+            "filters": {
+                "operator": "AND",
+                "conditions": [
+                    {"field": "meta.years", "operator": "==", "value": "2019"},
+                    {
+                        "field": "meta.companies",
+                        "operator": "in",
+                        "value": ["BMW", "Mercedes"],
+                    },
+                ],
+            },
+        },
+    },
+)
+```
+
+In Document Stores, the `filter_documents` method is used to apply filters to stored documents, if the specific integration supports filtering.
+
+The example below shows how filters can be passed to the `QdrantDocumentStore`:
+
+```python
+filters = {
+    "operator": "AND",
+    "conditions": [
+        {"field": "meta.type", "operator": "==", "value": "article"},
+        {"field": "meta.genre", "operator": "in", "value": ["economy", "politics"]},
+    ],
+}
+results = QdrantDocumentStore.filter_documents(filters=filters)
+```
+
+## Additional References
+
+:notebook: Tutorial: [Filtering Documents with Metadata](https://haystack.deepset.ai/tutorials/31_metadata_filtering)
+
+🧑‍🍳 Cookbook: [Extracting Metadata Filters from a Query](https://haystack.deepset.ai/cookbook/extracting_metadata_filters_from_a_user_query)

docs-website/docs/concepts/pipelines.mdx

@@ -0,0 +1,157 @@
+---
+title: "Pipelines"
+id: pipelines
+slug: "/pipelines"
+description: "To build modern search pipelines with LLMs, you need two things: powerful components and an easy way to put them together. The Haystack pipeline is built for this purpose and enables you to design and scale your interactions with LLMs."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+import YoutubeEmbed from "@site/src/components/YoutubeEmbed";
+
+# Pipelines
+
+To build modern search pipelines with LLMs, you need two things: powerful components and an easy way to put them together. The Haystack pipeline is built for this purpose and enables you to design and scale your interactions with LLMs.
+
+The pipelines in Haystack are directed multigraphs of different Haystack components and integrations. They give you the freedom to connect these components in various ways. This means that the pipeline doesn't need to be a continuous stream of information. With the flexibility of Haystack pipelines, you can have simultaneous flows, standalone components, loops, and other types of connections.
+
+## Flexibility
+
+Haystack pipelines are much more than just query and indexing pipelines. What a pipeline does, whether that be indexing, querying, fetching from an API, preprocessing or more, completely depends on how you design your pipeline and what components you use. While you can still create single-function pipelines, like indexing pipelines using ready-made components to clean up, split, and write the documents into a Document Store, or query pipelines that just take a query and return an answer, Haystack allows you to combine multiple use cases into one pipeline with decision components (like the `ConditionalRouter`) as well.
+
+### Agentic Pipelines
+
+Haystack loops and branches enable the creation of complex applications like agents. Here are a few examples on how to create them:
+
+- [Tutorial: Building a Chat Agent with Function Calling](https://haystack.deepset.ai/tutorials/40_building_chat_application_with_function_calling)
+- [Tutorial: Building an Agentic RAG with Fallback to Websearch](https://haystack.deepset.ai/tutorials/36_building_fallbacks_with_conditional_routing)
+- [Tutorial: Generating Structured Output with Loop-Based Auto-Correction](https://haystack.deepset.ai/tutorials/28_structured_output_with_loop)
+- [Cookbook: Define & Run Tools](https://haystack.deepset.ai/cookbook/tools_support)
+- [Cookbook: Conversational RAG using Memory](https://haystack.deepset.ai/cookbook/conversational_rag_using_memory)
+- [Cookbook: Newsletter Sending Agent with Experimental Haystack Tools](https://haystack.deepset.ai/cookbook/newsletter-agent)
+
+### Branching
+
+A pipeline can have multiple branches that process data concurrently. For example, to process different file types, you can have a pipeline with a bunch of converters, each handling a specific file type. You then feed all your files to the pipeline and it smartly divides and routes them to appropriate converters all at once, saving you the effort of sending your files one by one for processing.
+<ClickableImage src="/img/83f686b-Pipeline_Illustrations_1_1.png" alt="Pipeline architecture diagram showing components arranged in parallel branches that converge into a single pipeline flow" size="large" />
+
+### Loops
+
+Components in a pipeline can work in iterative loops, which you can cap at a desired number. This can be handy for scenarios like self-correcting loops, where you have a generator producing some output and then a validator component to check if the output is correct. If the generator's output has errors, the validator component can loop back to the generator for a corrected output. The loop goes on until the output passes the validation and can be sent further down the pipeline.
+
+See [Pipeline Loops](pipelines/pipeline-loops.mdx) for a deeper explanation of how loops are executed, how they terminate, and how to use them safely.
+
+<ClickableImage src="/img/2390eea-Pipeline_Illustrations_1_2.png" alt="Pipeline architecture diagram illustrating a feedback loop where output from later components loops back to earlier components" size="large" />
+
+### Async Execution and Streaming
+
+Pipelines execute components in parallel when their dependencies allow it. This improves performance in complex pipelines with independent operations. For example, a pipeline can run multiple Retrievers or LLM calls simultaneously, execute independent pipeline branches in parallel, and efficiently handle I/O-bound operations that would otherwise cause delays. You can cap the number of components running at the same time with the `concurrency_limit` init parameter.
+
+Besides the blocking `run` method, every pipeline offers three ways to run asynchronously:
+
+- `run_async`: Executes the pipeline in a single non-blocking call, ideal for integrating a pipeline into a larger async application or service.
+- `run_async_generator`: Yields partial outputs as components complete their tasks, which is useful for monitoring progress, debugging, and handling outputs incrementally.
+- `stream`: Runs the pipeline and returns a handle that streams [`StreamingChunk`](/reference/data-classes-api#streamingchunk) objects as they are produced — a convenient way to stream LLM output from an async application, such as an API endpoint. Iterate the handle with `async for` to consume the chunks; after iteration ends, `handle.result` holds the final pipeline output (the same dictionary returned by `run_async`).
+
+```python
+import asyncio
+
+from haystack import Pipeline
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.dataclasses import ChatMessage
+
+pipe = Pipeline()
+pipe.add_component(
+    "prompt_builder",
+    ChatPromptBuilder(template=[ChatMessage.from_user("Tell me about {{topic}}")]),
+)
+pipe.add_component("llm", OpenAIChatGenerator())
+pipe.connect("prompt_builder.prompt", "llm.messages")
+
+
+async def main():
+    handle = pipe.stream(data={"prompt_builder": {"topic": "Italy"}})
+    async for chunk in handle:
+        print(chunk.content, end="", flush=True)
+    return handle.result
+
+
+result = asyncio.run(main())
+```
+
+By default, chunks from every streaming-capable component are forwarded; pass `streaming_components` with a list of component names to stream only specific components. If the consumer abandons iteration, the underlying pipeline run is cancelled automatically; pass `cancel_on_abandon=False` to let it run to completion instead.
+
+If a `streaming_callback` is set on a component (at init or at runtime through `data`), it is still invoked for each chunk in addition to the chunks being pushed to the handle. When streaming, components accept a sync `streaming_callback` in `run_async` too — see the [Choosing the Right Generator guide](../pipeline-components/generators/guides-to-generators/choosing-the-right-generator.mdx#sync-and-async-callbacks) for details.
+
+#### Error Handling and Task Cancellation
+
+If a component raises an error while sibling components are still running concurrently, the pipeline cancels and drains those in-flight tasks before re-raising the original error, so no tasks keep running in the background. The same cleanup applies when you stop iterating `run_async_generator` early (for example, by breaking out of the loop and closing the generator) or when the run itself is cancelled.
+
+Note that cancellation only interrupts components that run natively async. Sync components are offloaded to a worker thread, which cannot be interrupted and runs to completion in the background. Their outputs are discarded, so the pipeline state stays consistent, but the component's side effects still complete.
+
+## SuperComponents
+
+To simplify your code, we have introduced [SuperComponents](components/supercomponents.mdx) that allow you to wrap complete pipelines and reuse them as a single component. Check out their documentation page for the details and examples.
+
+## Data Flow
+
+While the data (the initial query) flows through the entire pipeline, individual values are only passed from one component to another when they are connected. Therefore, not all components have access to all the data. This approach offers the benefits of speed and ease of debugging.
+
+To connect components and integrations in a pipeline, you must know the names of their inputs and outputs. The output of one component must be accepted as input by the following component. When you connect components in a pipeline with `Pipeline.connect()`, it validates if the input and output types match.
+
+### Smart Pipeline Connections
+
+Pipelines support smarter connection semantics that simplify how components are wired together.
+
+Compatible outputs can be implicitly combined when connected to a single input.
+Pipelines also perform implicit type adaptation at connection time for some selected types.
+
+These behaviors reduce the need for glue components like `Joiners` and `OutputAdapters`, keeping pipelines concise and easier to read.
+
+See [Smart Pipeline Connections](pipelines/smart-pipeline-connections.mdx) for details and examples.
+
+<YoutubeEmbed videoId="SxAwyeCkguc" title="Introduction to Haystack Pipelines" />
+
+## Steps to Create a Pipeline Explained
+
+Once all your components are created and ready to be combined in a pipeline, there are four steps to make it work:
+
+1. Create the pipeline with `Pipeline()`.
+   This creates the Pipeline object.
+2. Add components to the pipeline, one by one, with `.add_component(name, component)`.
+   This just adds components to the pipeline without connecting them yet. It's especially useful for loops as it allows the smooth connection of the components in the next step because they all already exist in the pipeline.
+3. Connect components with `.connect("producer_component.output_name", "consumer_component.input_name")`.
+   At this step, you explicitly connect one of the outputs of a component to one of the inputs of the next component. This is also when the pipeline validates the connection without running the components. It makes the validation fast.
+4. Run the pipeline with `.run({"component_1": {"mandatory_inputs": value}})`.
+   Finally, you run the Pipeline by specifying the first component in the pipeline and passing its mandatory inputs. Optionally, you can pass inputs to other components, for example: `.run({"component_1": {"mandatory_inputs": value}, "component_2": {"inputs": value}})`.
+
+The full pipeline [example](pipelines/creating-pipelines.mdx#example) in [Creating Pipelines](pipelines/creating-pipelines.mdx) shows how all the elements come together to create a working RAG pipeline.
+
+Once you create your pipeline, you can [visualize it in a graph](pipelines/visualizing-pipelines.mdx) to understand how the components are connected and make sure that's how you want them. You can use Mermaid graphs to do that.
+
+## Validation
+
+Validation happens when you connect pipeline components with `.connect()`, but before running the components to make it faster. The pipeline validates that:
+
+- The components exist in the pipeline.
+- The components' outputs and inputs match and are explicitly indicated. For example, if a component produces two outputs, when connecting it to another component, you must indicate which output connects to which input.
+- The components' types match.
+- For input types other than `Variadic`, checks if the input is already occupied by another connection.
+
+All of these checks produce detailed errors to help you quickly fix any issues identified.
+
+## Serialization
+
+Thanks to serialization, you can save and then load your pipelines. Serialization is converting a Haystack pipeline into a format you can store on disk or send over the wire. It's particularly useful for:
+
+- Editing, storing, and sharing pipelines.
+- Modifying existing pipelines in a format different than Python.
+
+Haystack pipelines delegate the serialization to its components, so serializing a pipeline simply means serializing each component in the pipeline one after the other, along with their connections. The pipeline is serialized into a dictionary format, which acts as an intermediate format that you can then convert into the final format you want.
+
+:::info[Serialization formats]
+
+Haystack only supports YAML format at this time. We'll be rolling out more formats gradually.
+:::
+
+For serialization to be possible, components must support conversion from and to Python dictionaries. All Haystack components have two methods that make them serializable: `from_dict` and `to_dict`. The `Pipeline` class, in turn, has its own `from_dict` and `to_dict` methods that take care of serializing components and connections.

docs-website/docs/concepts/pipelines/creating-pipelines.mdx

@@ -0,0 +1,258 @@
+---
+title: "Creating Pipelines"
+id: creating-pipelines
+slug: "/creating-pipelines"
+description: "Learn the general principles of creating a pipeline."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+
+# Creating Pipelines
+
+Learn the general principles of creating a pipeline.
+
+You can use these instructions to create both indexing and query pipelines.
+
+This task uses an example of a semantic document search pipeline.
+
+## Prerequisites
+
+For each component you want to use in your pipeline, you must know the names of its input and output. You can check them on the documentation page for a specific component or in the component's `run()` method. For more information, see [Components: Input and Output](../components.mdx#input-and-output).
+
+## Steps to Create a Pipeline
+
+### 1\. Import dependencies
+
+Import all the dependencies, like pipeline, documents, Document Store, and all the components you want to use in your pipeline.
+For example, to create a semantic document search pipelines, you need the `Document` object, the pipeline, the Document Store, Embedders, and a Retriever:
+
+The examples on this page use Sentence Transformers embedders that have moved to the `sentence-transformers-haystack` package. Install it to run the examples:
+
+```shell
+pip install sentence-transformers-haystack
+```
+
+```python
+from haystack import Document, Pipeline
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack_integrations.components.embedders.sentence_transformers import (
+    SentenceTransformersTextEmbedder,
+)
+from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+```
+
+### 2\. Initialize components
+
+Initialize the components, passing any parameters you want to configure:
+
+```python
+document_store = InMemoryDocumentStore(embedding_similarity_function="cosine")
+text_embedder = SentenceTransformersTextEmbedder()
+retriever = InMemoryEmbeddingRetriever(document_store=document_store)
+```
+
+### 3\. Create the pipeline
+
+```python
+query_pipeline = Pipeline()
+```
+
+### 4\. Add components
+
+Add components to the pipeline one by one. The order in which you do this doesn't matter:
+
+```python
+query_pipeline.add_component("component_name", component_type)
+
+# Here is an example of how you'd add the components initialized in step 2 above:
+query_pipeline.add_component("text_embedder", text_embedder)
+query_pipeline.add_component("retriever", retriever)
+
+# You could also add components without initializing them before:
+query_pipeline.add_component("text_embedder", SentenceTransformersTextEmbedder())
+query_pipeline.add_component(
+    "retriever",
+    InMemoryEmbeddingRetriever(document_store=document_store),
+)
+```
+
+### 5\. Connect components
+
+Connect the components by indicating which output of a component should be connected to the input of the next component. If a component has only one input or output and the connection is obvious, you can just pass the component name without specifying the input or output.
+
+To understand what inputs are expected to run your pipeline, use an `.inputs()` pipeline function. See a detailed examples in the [Pipeline Inputs](#pipeline-inputs) section below.
+
+Here's a more visual explanation within the code:
+
+```python
+# This is the syntax to connect components. Here you're connecting output1 of component1 to input1 of component2:
+pipeline.connect("component1.output1", "component2.input1")
+
+# If both components have only one output and input, you can just pass their names:
+pipeline.connect("component1", "component2")
+
+# If one of the components has only one output but the other has multiple inputs,
+# you can pass just the name of the component with a single output, but for the component with
+# multiple inputs, you must specify which input you want to connect
+
+# Here, component1 has only one output, but component2 has multiple inputs:
+pipeline.connect("component1", "component2.input1")
+
+# And here's how it should look like for the semantic document search pipeline we're using as an example:
+pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
+# Because the InMemoryEmbeddingRetriever only has one input, this is also correct:
+pipeline.connect("text_embedder.embedding", "retriever")
+```
+
+You need to link all the components together, connecting them gradually in pairs. Here's an explicit example for the pipeline we're assembling:
+
+```python
+# Imagine this pipeline has four components: text_embedder, retriever, prompt_builder and llm.
+# Here's how you would connect them into a pipeline:
+
+query_pipeline.connect("text_embedder.embedding", "retriever")
+query_pipeline.connect("retriever", "prompt_builder.documents")
+query_pipeline.connect("prompt_builder", "llm")
+```
+
+### 6\. Run the pipeline
+
+Wait for the pipeline to validate the components and connections. If everything is OK, you can now run the pipeline. `Pipeline.run()` can be called in two ways, either passing a dictionary of the component names and their inputs, or by directly passing just the inputs. When passed directly, the pipeline resolves inputs to the correct components.
+
+```python
+# Here's one way of calling the run() method
+results = pipeline.run({"component1": {"input1_value": value1, "input2_value": value2}})
+
+# The inputs can also be passed directly without specifying component names
+results = pipeline.run({"input1_value": value1, "input2_value": value2})
+
+# This is how you'd run the semantic document search pipeline we're using as an example:
+query = "Here comes the query text"
+results = query_pipeline.run({"text_embedder": {"text": query}})
+```
+
+## Pipeline Inputs
+
+If you need to understand what component inputs are expected to run your pipeline, Haystack features a useful pipeline function `.inputs()` that lists all the required inputs for the components.
+
+This is how it works:
+
+```python
+# A short pipeline example that converts webpages into documents
+from haystack import Pipeline
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.fetchers import LinkContentFetcher
+from haystack.components.converters import HTMLToDocument
+from haystack.components.writers import DocumentWriter
+
+document_store = InMemoryDocumentStore()
+fetcher = LinkContentFetcher()
+converter = HTMLToDocument()
+writer = DocumentWriter(document_store=document_store)
+
+pipeline = Pipeline()
+pipeline.add_component(instance=fetcher, name="fetcher")
+pipeline.add_component(instance=converter, name="converter")
+pipeline.add_component(instance=writer, name="writer")
+
+pipeline.connect("fetcher.streams", "converter.sources")
+pipeline.connect("converter.documents", "writer.documents")
+
+# Requesting a list of required inputs
+pipeline.inputs()
+
+# {'fetcher': {'urls': {'type': typing.List[str], 'is_mandatory': True}},
+# 'converter': {'meta': {'type': typing.Union[typing.Dict[str, typing.Any], typing.List[typing.Dict[str, typing.Any]], NoneType],
+# 'is_mandatory': False,
+# 'default_value': None},
+# 'extraction_kwargs': {'type': typing.Optional[typing.Dict[str, typing.Any]],
+# 'is_mandatory': False,
+# 'default_value': None}},
+# 'writer': {'policy': {'type': typing.Optional[haystack.document_stores.types.policy.DuplicatePolicy],
+# 'is_mandatory': False,
+# 'default_value': None}}}
+```
+
+From the above response, you can see that the `urls` input is mandatory for `LinkContentFetcher`. This is how you would then run this pipeline:
+
+```python
+pipeline.run(
+    data={"fetcher": {"urls": ["https://docs.haystack.deepset.ai/docs/pipelines"]}},
+)
+```
+
+## Example
+
+The following example walks you through creating a RAG pipeline.
+
+```python
+# import necessary dependencies
+from haystack import Pipeline, Document
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.retrievers import InMemoryBM25Retriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.builders import ChatPromptBuilder
+from haystack.utils import Secret
+from haystack.dataclasses import ChatMessage
+
+# create a document store and write documents to it
+document_store = InMemoryDocumentStore()
+document_store.write_documents(
+    [
+        Document(content="My name is Jean and I live in Paris."),
+        Document(content="My name is Mark and I live in Berlin."),
+        Document(content="My name is Giorgio and I live in Rome."),
+    ],
+)
+
+# A prompt corresponds to an NLP task and contains instructions for the model. Here, the pipeline will go through each Document to figure out the answer.
+prompt_template = [
+    ChatMessage.from_system(
+        """
+        Given these documents, answer the question.
+        Documents:
+        {% for doc in documents %}
+            {{ doc.content }}
+        {% endfor %}
+        Question:
+        """,
+    ),
+    ChatMessage.from_user("{{question}}"),
+    ChatMessage.from_system("Answer:"),
+]
+
+# create the components adding the necessary parameters
+retriever = InMemoryBM25Retriever(document_store=document_store)
+prompt_builder = ChatPromptBuilder(template=prompt_template, required_variables="*")
+llm = OpenAIChatGenerator(
+    api_key=Secret.from_env_var("OPENAI_API_KEY"),
+    model="gpt-4o-mini",
+)
+
+# Create the pipeline and add the components to it. The order doesn't matter.
+# At this stage, the Pipeline validates the components without running them yet.
+rag_pipeline = Pipeline()
+rag_pipeline.add_component("retriever", retriever)
+rag_pipeline.add_component("prompt_builder", prompt_builder)
+rag_pipeline.add_component("llm", llm)
+
+# Arrange pipeline components in the order you need them. If a component has more than one inputs or outputs, indicate which input you want to connect to which output using the format ("component_name.output_name", "component_name, input_name").
+rag_pipeline.connect("retriever", "prompt_builder.documents")
+rag_pipeline.connect("prompt_builder", "llm")
+
+# Run the pipeline by specifying the first component in the pipeline and passing its mandatory inputs. Optionally, you can pass inputs to other components.
+question = "Who lives in paris?"
+results = rag_pipeline.run(
+    {
+        "retriever": {"query": question},
+        "prompt_builder": {"question": question},
+    },
+)
+
+print(results["llm"]["replies"])
+```
+
+Here's what a [visualized Mermaid graph](visualizing-pipelines.mdx) of this pipeline would look like:
+
+<br />
+<ClickableImage src="/img/vizualised-rag-pipeline.png" alt="RAG pipeline diagram with three connected components: InMemoryBM25Retriever receives a query string and outputs documents, ChatPromptBuilder combines the documents with a question input to create prompt messages, and OpenAIChatGenerator processes the messages to produce replies. Each component box displays its class name and optional input parameters." size="large" />

docs-website/docs/concepts/pipelines/debugging-pipelines.mdx

@@ -0,0 +1,125 @@
+---
+title: "Debugging Pipelines"
+id: debugging-pipelines
+slug: "/debugging-pipelines"
+description: "Learn how to debug and troubleshoot your Haystack pipelines."
+---
+
+import ClickableImage from "@site/src/components/ClickableImage";
+
+# Debugging Pipelines
+
+Learn how to debug and troubleshoot your Haystack pipelines.
+
+There are several options available to you to debug your pipelines:
+
+- [Inspect your components' outputs](#inspecting-component-outputs)
+- [Adjust logging](#logging)
+- [Set up tracing](#tracing)
+- [Try one of the monitoring tool integrations](#monitoring-tools)
+
+## Inspecting Component Outputs
+
+To view outputs from specific pipeline components, add the `include_outputs_from` parameter when executing your pipeline. Place it after the input dictionary and set it to the name of the component whose output you want included in the result.
+
+For example, here’s how you can print the output of `PromptBuilder` in this pipeline:
+
+```python
+from haystack import Pipeline, Document
+from haystack.utils import Secret
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder
+from haystack.dataclasses import ChatMessage
+
+# Documents
+documents = [
+    Document(content="Joe lives in Berlin"),
+    Document(content="Joe is a software engineer"),
+]
+
+# Define prompt template
+prompt_template = [
+    ChatMessage.from_system("You are a helpful assistant."),
+    ChatMessage.from_user(
+        "Given these documents, answer the question.\nDocuments:\n"
+        "{% for doc in documents %}{{ doc.content }}{% endfor %}\n"
+        "Question: {{query}}\nAnswer:",
+    ),
+]
+
+# Define pipeline
+p = Pipeline()
+p.add_component(
+    instance=ChatPromptBuilder(
+        template=prompt_template,
+        required_variables={"query", "documents"},
+    ),
+    name="prompt_builder",
+)
+p.add_component(
+    instance=OpenAIChatGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")),
+    name="llm",
+)
+p.connect("prompt_builder", "llm.messages")
+
+# Define question
+question = "Where does Joe live?"
+
+# Execute pipeline
+result = p.run(
+    {"prompt_builder": {"documents": documents, "query": question}},
+    include_outputs_from="prompt_builder",
+)
+
+# Print result
+print(result)
+```
+
+## Logging
+
+Adjust the logging format according to your debugging needs. See our [Logging](../../development/logging.mdx) documentation for details.
+
+## Real-Time Pipeline Logging
+
+Use Haystack's [`LoggingTracer`](https://github.com/deepset-ai/haystack/blob/main/haystack/tracing/logging_tracer.py) logs to inspect the data that's flowing through your pipeline in real-time.
+
+This feature is particularly helpful during experimentation and prototyping, as you don’t need to set up any tracing backend beforehand.
+
+Here’s how you can enable this tracer. In this example, we are adding color tags (this is optional) to highlight the components' names and inputs:
+
+```python
+import logging
+from haystack import tracing
+from haystack.tracing.logging_tracer import LoggingTracer
+
+logging.basicConfig(
+    format="%(levelname)s - %(name)s -  %(message)s",
+    level=logging.WARNING,
+)
+logging.getLogger("haystack").setLevel(logging.DEBUG)
+
+tracing.tracer.is_content_tracing_enabled = (
+    True  # to enable tracing/logging content (inputs/outputs)
+)
+tracing.enable_tracing(
+    LoggingTracer(
+        tags_color_strings={
+            "haystack.component.input": "\x1b[1;31m",
+            "haystack.component.name": "\x1b[1;34m",
+        },
+    ),
+)
+```
+
+Here’s what the resulting log would look like when a pipeline is run:
+<ClickableImage src="/img/55c3d5c84282d726c95fb3350ec36be49a354edca8a6164f5dffdab7121cec58-image_2.png" alt="Console output showing Haystack pipeline execution with DEBUG level tracing logs including component names, types, and input/output specifications" />
+
+## Tracing
+
+To get a bigger picture of the pipeline’s performance, try tracing it with [Langfuse](../../development/tracing/langfuse.mdx).
+
+Our [Tracing](../../development/tracing.mdx) page has more about other tracing solutions for Haystack.
+
+## Monitoring Tools
+
+Take a look at available tracing and monitoring [integrations](https://haystack.deepset.ai/integrations?type=Monitoring+Tool&version=2.0) for Haystack pipelines, such as Arize AI or Arize Phoenix.

docs-website/docs/concepts/pipelines/pipeline-breakpoints.mdx

@@ -0,0 +1,160 @@
+---
+title: "Pipeline Breakpoints"
+id: pipeline-breakpoints
+slug: "/pipeline-breakpoints"
+description: "Learn how to pause and resume Haystack pipeline execution using breakpoints to debug, inspect, and continue workflows from saved snapshots."
+---
+
+# Pipeline Breakpoints
+
+Learn how to pause and resume Haystack pipeline execution using breakpoints to debug, inspect, and continue workflows from saved snapshots.
+
+## Introduction
+
+Haystack pipelines support breakpoints for debugging complex execution flows. A `Breakpoint` allows you to pause the execution at specific components, inspect the pipeline state, and resume execution from saved snapshots. This feature works for any regular component as well as an `Agent` component.
+
+You can set a `Breakpoint` on any component in a pipeline with a specific visit count. When triggered, the system stops the execution of the `Pipeline` and captures a snapshot of the current pipeline state. The state can be saved to a JSON file when snapshot file saving is enabled, see [Snapshot file saving](#snapshot-file-saving) below. You can inspect and modify the snapshot and use it to resume execution from the exact point where it stopped.
+
+## Setting a `Breakpoint` on a Regular Component
+
+Create a `Breakpoint` by specifying the component name and the visit count at which to trigger it. This is useful for pipelines with loops. The default `visit_count` value is 0.
+
+```python
+from haystack.dataclasses.breakpoints import Breakpoint
+from haystack.core.errors import BreakpointException
+
+# Create a breakpoint that triggers on the first visit to the "llm" component
+break_point = Breakpoint(
+    component_name="llm",
+    visit_count=0,  # 0 = first visit, 1 = second visit, etc.
+    snapshot_file_path="/path/to/snapshots",  # Optional: save snapshot to file
+)
+
+# Run pipeline with breakpoint
+try:
+    result = pipeline.run(data=input_data, break_point=break_point)
+except BreakpointException as e:
+    print(f"Breakpoint triggered at component: {e.component}")
+    print(f"Component inputs: {e.inputs}")
+    print(f"Pipeline results so far: {e.results}")
+```
+
+A `BreakpointException` is raised containing the component inputs and the outputs of the pipeline up until the moment where the execution was interrupted, such as just before the execution of component associated with the breakpoint – the `llm` in the example above.
+
+If a `snapshot_file_path` is specified in the `Breakpoint` and snapshot file saving is enabled, the system saves a JSON snapshot with the same information as in the `BreakpointException`. Snapshot file saving to disk is disabled by default; see [Snapshot file saving](#snapshot-file-saving) below.
+
+To access the pipeline state during the breakpoint we can both catch the exception raised by the breakpoint as well as specify where the JSON file should be saved, note that file saving is enabled must be enabled.
+
+## Using a custom snapshot callback
+
+You can pass a `snapshot_callback` to `Pipeline.run()` to handle snapshots yourself instead of saving to a file. When a breakpoint is triggered or a snapshot is created on error, the callback is invoked with the `PipelineSnapshot` object. This is useful for saving snapshots to a database, sending them to a remote service, or custom logging.
+
+```python
+from haystack.core.errors import BreakpointException
+from haystack.dataclasses.breakpoints import Breakpoint, PipelineSnapshot
+
+
+def my_snapshot_callback(snapshot: PipelineSnapshot) -> None:
+    # Custom handling: e.g. save to DB, send to API, or log
+    print(f"Snapshot at component: {snapshot.break_point}")
+
+
+break_point = Breakpoint(component_name="llm", visit_count=0)
+try:
+    result = pipeline.run(
+        data=input_data,
+        break_point=break_point,
+        snapshot_callback=my_snapshot_callback,
+    )
+except BreakpointException as e:
+    print(f"Breakpoint triggered: {e.component}")
+```
+
+When `snapshot_callback` is provided, file-saving is skipped and the callback is responsible for handling the snapshot.
+
+## Snapshot file saving
+
+Snapshot file saving to disk is **disabled by default**. To save snapshots as JSON files when a breakpoint is triggered or on pipeline failure, set the environment variable `HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED` to `"true"` or `"1"` (case-insensitive). When enabled, snapshots are written to the path given by `snapshot_file_path` on the breakpoint, or to the default directory in [Error Recovery with Snapshots](#error-recovery-with-snapshots) when a run fails.
+
+Custom `snapshot_callback` functions are always invoked when provided, regardless of this setting.
+
+```python
+import os
+
+# Enable saving snapshot files to disk
+os.environ["HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED"] = "true"
+
+break_point = Breakpoint(
+    component_name="llm",
+    visit_count=0,
+    snapshot_file_path="/path/to/snapshots",
+)
+# When the breakpoint triggers, a JSON file will be written to /path/to/snapshots/
+```
+
+## Resuming a Pipeline Execution from a Breakpoint
+
+To resume the execution of a pipeline from the breakpoint, pass the path to the generated JSON file at the run time of the pipeline, using the `pipeline_snapshot`.
+
+Use the `load_pipeline_snapshot()` to first load the JSON and then pass it to the pipeline.
+
+```python
+from haystack.core.pipeline.breakpoint import load_pipeline_snapshot
+
+# Load the snapshot
+snapshot = load_pipeline_snapshot("llm_2025_05_03_11_23_23.json")
+
+# Resume execution from the snapshot
+result = pipeline.run(data={}, pipeline_snapshot=snapshot)
+print(result["llm"]["replies"])
+```
+
+## Error Recovery with Snapshots
+
+Pipelines automatically create a snapshot of the last valid state if a run fails. The snapshot contains inputs, visit counts, and intermediate outputs up to the failure. You can inspect it, fix the issue, and resume execution from that checkpoint instead of restarting the whole run.
+
+### Access the Snapshot on Failure
+
+Wrap `pipeline.run()` in a `try`/`except` block and retrieve the snapshot from the raised `PipelineRuntimeError`:
+
+```python
+from haystack.core.errors import PipelineRuntimeError
+
+try:
+    pipeline.run(data=input_data)
+except PipelineRuntimeError as e:
+    snapshot = e.pipeline_snapshot
+    if snapshot is not None:
+        intermediate_outputs = snapshot.pipeline_state.pipeline_outputs
+        # Inspect intermediate_outputs to diagnose the failure
+```
+
+When snapshot file saving is enabled (see [Snapshot file saving](#snapshot-file-saving)), Haystack also saves the same snapshot as a JSON file on disk.
+The directory is chosen automatically in this order:
+
+- `~/.haystack/pipeline_snapshot`
+- `/tmp/haystack/pipeline_snapshot`
+- `./.haystack/pipeline_snapshot`
+
+Filenames will have the following pattern: `{component_name}_{visit_nr}_{YYYY_MM_DD_HH_MM_SS}.json`.
+
+### Resume from a Snapshot
+
+You can resume directly from the in-memory snapshot or load it from disk.
+
+Resume from memory:
+
+```python
+result = pipeline.run(data={}, pipeline_snapshot=snapshot)
+```
+
+Resume from disk:
+
+```python
+from haystack.core.pipeline.breakpoint import load_pipeline_snapshot
+
+snapshot = load_pipeline_snapshot(
+    "/path/to/.haystack/pipeline_snapshot/reader_0_2025_09_20_12_33_10.json",
+)
+result = pipeline.run(data={}, pipeline_snapshot=snapshot)
+```

docs-website/docs/concepts/pipelines/pipeline-loops.mdx

@@ -0,0 +1,253 @@
+---
+title: "Pipeline Loops"
+id: pipeline-loops
+slug: "/pipeline-loops"
+description: "Understand how loops work in Haystack pipelines, how they terminate, and how to use them safely for feedback and self-correction."
+---
+
+# Pipeline Loops
+
+Learn how loops work in Haystack pipelines, how they terminate, and how to use them for feedback and self-correction.
+
+Haystack pipelines support **loops**: cycles in the component graph where the output of a later component is fed back into an earlier one.
+This enables feedback flows such as self-correction, validation, or iterative refinement, as well as more advanced [agentic behavior](../pipelines.mdx#agentic-pipelines).
+
+At runtime, the pipeline re-runs a component whenever all of its required inputs are ready again.
+You control when loops stop either by designing your graph and routing logic carefully or by using built-in [safety limits](#loop-termination-and-safety-limits).
+
+## Multiple Runs of the Same Component
+
+If a component participates in a loop, it can be run multiple times within a single `Pipeline.run()` call.
+The pipeline keeps an internal visit counter for each component:
+
+- Each time the component runs, its visit count increases by 1.
+- You can use this visit count in debugging tools like [breakpoints](./pipeline-breakpoints.mdx) to inspect specific iterations of a loop.
+
+In the final pipeline result:
+
+- For each component that ran, the pipeline returns **only the last-produced output**.
+- To capture outputs from intermediate components (for example, a validator or a router) in the final result dictionary, use the `include_outputs_from` argument of `Pipeline.run()`.
+
+## Loop Termination and Safety Limits
+
+Loops must eventually stop so that a pipeline run can complete.
+There are two main ways a loop ends:
+
+1. **Natural completion**: No more components are runnable  
+   The pipeline finishes when the work queue is empty and no component can run again (for example, the router stops feeding inputs back into the loop).
+
+2. **Reaching the maximum run count**  
+   Every pipeline has a per-component run limit, controlled by the `max_runs_per_component` parameter of the `Pipeline` constructor, which is `100` by default. If any component exceeds this limit, Haystack raises a `PipelineMaxComponentRuns` error.
+
+   You can set this limit to a lower value:
+
+   ```python
+   from haystack import Pipeline
+
+   pipe = Pipeline(max_runs_per_component=5)
+   ```
+
+   The limit is checked before each execution, so a component with a limit of 3 will complete 3 runs successfully before the error is raised on the 4th attempt.
+
+   This safeguard is especially important when experimenting with new loops or complex routing logic.
+   If your loop condition is wrong or never satisfied, the error prevents the pipeline from running indefinitely.
+
+## Example: Feedback Loop for Self-Correction
+
+The following example shows a simple feedback loop where:
+
+- A `ChatPromptBuilder` creates a prompt that includes previous incorrect replies.
+- An `OpenAIChatGenerator` produces an answer.
+- A `ConditionalRouter` checks if the answer is correct:
+  - If correct, it sends the answer to `final_answer` and the loop ends.
+  - If incorrect, it sends the answer back to the `ChatPromptBuilder`, which triggers another iteration.
+
+```python
+from haystack import Pipeline
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.routers import ConditionalRouter
+from haystack.dataclasses import ChatMessage
+
+template = [
+    ChatMessage.from_system(
+        "Answer the following question concisely with just the answer, no punctuation.",
+    ),
+    ChatMessage.from_user(
+        "{% if previous_replies %}"
+        "Previously you replied incorrectly: {{ previous_replies[0].text }}\n"
+        "{% endif %}"
+        "Question: {{ query }}",
+    ),
+]
+
+prompt_builder = ChatPromptBuilder(template=template, required_variables=["query"])
+generator = OpenAIChatGenerator()
+
+router = ConditionalRouter(
+    routes=[
+        {
+            # End the loop when the answer is correct
+            "condition": "{{ 'Rome' in replies[0].text }}",
+            "output": "{{ replies }}",
+            "output_name": "final_answer",
+            "output_type": list[ChatMessage],
+        },
+        {
+            # Loop back when the answer is incorrect
+            "condition": "{{ 'Rome' not in replies[0].text }}",
+            "output": "{{ replies }}",
+            "output_name": "previous_replies",
+            "output_type": list[ChatMessage],
+        },
+    ],
+    unsafe=True,  # Required to handle ChatMessage objects
+)
+
+pipe = Pipeline(max_runs_per_component=3)
+
+pipe.add_component("prompt_builder", prompt_builder)
+pipe.add_component("generator", generator)
+pipe.add_component("router", router)
+
+pipe.connect("prompt_builder.prompt", "generator.messages")
+pipe.connect("generator.replies", "router.replies")
+pipe.connect("router.previous_replies", "prompt_builder.previous_replies")
+
+result = pipe.run(
+    {
+        "prompt_builder": {
+            "query": "What is the capital of Italy? If the statement 'Previously you replied incorrectly:' is missing "
+            "above then answer with Milan.",
+        },
+    },
+    include_outputs_from={"router", "prompt_builder"},
+)
+
+print(result["prompt_builder"]["prompt"][1].text)  # Shows the last prompt used
+print(result["router"]["final_answer"][0].text)  # Rome
+```
+
+### What Happens During This Loop
+
+1. **First iteration**
+   - `prompt_builder` runs with `query="What is the capital of Italy?"` and no previous replies.
+   - `generator` returns a `ChatMessage` with the LLM's answer.
+   - The router evaluates its conditions and checks if `"Rome"` is in the reply.
+   - If the answer is incorrect, `previous_replies` is fed back into `prompt_builder.previous_replies`.
+
+2. **Subsequent iterations** (if needed)
+   - `prompt_builder` runs again, now including the previous incorrect reply in the user message.
+   - `generator` produces a new answer with the additional context.
+   - The router checks again whether the answer contains `"Rome"`.
+
+3. **Termination**
+   - When the router routes to `final_answer`, no more inputs are fed back into the loop.
+   - The queue empties and the pipeline run finishes successfully.
+
+Because we used `max_runs_per_component=3`, any unexpected behavior that causes the loop to continue would raise a `PipelineMaxComponentRuns` error instead of looping forever.
+
+## Components for Building Loops
+
+Two components are particularly useful for building loops:
+
+- **[`ConditionalRouter`](../../pipeline-components/routers/conditionalrouter.mdx)**: Routes data to different outputs based on conditions. Use it to decide whether to exit the loop or continue iterating. The example above uses this pattern.
+
+- **[`BranchJoiner`](../../pipeline-components/joiners/branchjoiner.mdx)**: Merges inputs from multiple sources into a single output. Use it when a component inside the loop needs to receive both the initial input (on the first iteration) and looped-back values (on subsequent iterations). For example, you might use `BranchJoiner` to feed both user input and validation errors into the same Generator. See the [BranchJoiner documentation](../../pipeline-components/joiners/branchjoiner.mdx#enabling-loops) for a complete loop example.
+
+## Greedy vs. Lazy Variadic Sockets in Loops
+
+Some components support variadic inputs that can receive multiple values on a single socket.
+In loops, variadic behavior controls how inputs are consumed across iterations.
+
+- **Greedy variadic sockets**  
+  Consume exactly one value at a time and remove it after the component runs.
+  This includes user-provided inputs, which prevents them from retriggering the component indefinitely.
+  Most variadic sockets are greedy by default.
+
+- **Lazy variadic sockets**  
+  Accumulate all values received from predecessors across iterations.
+  Useful when you need to collect multiple partial results over time (for example, gathering outputs from several loop iterations before proceeding).
+
+For most loop scenarios it's sufficient to just connect components as usual and use `max_runs_per_component` to protect against mistakes.
+
+## Troubleshooting Loops
+
+If your pipeline seems stuck or runs longer than expected, here are common causes and how to debug them.
+
+### Common Causes of Infinite Loops
+
+1. **Condition never satisfied**: Your exit condition (for example, `"Rome" in reply`) might never be true due to LLM behavior or data issues. Always set a reasonable `max_runs_per_component` as a safety net.
+
+2. **Relying on optional outputs**: When a component has multiple output sockets but only returns some of them, the unreturned outputs don't trigger their downstream connections. This can cause confusion in loops.
+
+   For example, this pattern can be problematic:
+
+   ```python
+   @component
+   class Validator:
+       @component.output_types(valid=str, invalid=Optional[str])
+       def run(self, text: str):
+           if is_valid(text):
+               return {"valid": text}  # "invalid" is never returned
+           else:
+               return {"invalid": text}
+   ```
+
+   If you connect `invalid` back to an upstream component for retry, but also have other connections that keep the loop alive, you might get unexpected behavior.
+
+   Instead, use a `ConditionalRouter` with explicit, mutually exclusive conditions:
+
+   ```python
+   router = ConditionalRouter(
+       routes=[
+           {"condition": "{{ is_valid }}", "output": "{{ text }}", "output_name": "valid", ...},
+           {"condition": "{{ not is_valid }}", "output": "{{ text }}", "output_name": "invalid", ...},
+       ]
+   )
+   ```
+
+3. **User inputs retriggering the loop**: If a user-provided input is connected to a socket inside the loop, it might cause the loop to restart unexpectedly.
+
+   ```python
+   # Problematic: user input goes directly to a component inside the loop
+   result = pipe.run({
+       "generator": {"prompt": query},  # This input persists and may retrigger the loop
+   })
+
+   # Better: use an entry-point component outside the loop
+   result = pipe.run({
+       "prompt_builder": {"query": query},  # Entry point feeds into the loop once
+   })
+   ```
+
+   See [Greedy vs. Lazy Variadic Sockets](#greedy-vs-lazy-variadic-sockets-in-loops) for details on how inputs are consumed.
+
+4. **Multiple paths feeding the same component**: If a component inside the loop receives inputs from multiple sources, it runs whenever *any* path provides input.
+
+   ```python
+   # Component receives from two sources – runs when either provides input
+   pipe.connect("source_a.output", "processor.input")
+   pipe.connect("source_b.output", "processor.input")  # Variadic input
+   ```
+
+   Ensure you understand when each path produces output, or use `BranchJoiner` to explicitly control the merge point.
+
+### Debugging Tips
+
+1. **Start with a low limit**: When developing loops, set `max_runs_per_component=3` or similar. This helps you catch issues early with a clear error instead of waiting for a timeout.
+
+2. **Use `include_outputs_from`**: Add intermediate components (like your router) to see what's happening at each step:
+   ```python
+   result = pipe.run(data, include_outputs_from={"router", "validator"})
+   ```
+
+3. **Enable tracing**: Use tracing to see every component execution, including inputs and outputs. This makes it easy to follow each iteration of the loop. For quick debugging, use `LoggingTracer` ([setup instructions](./debugging-pipelines.mdx#real-time-pipeline-logging)). For deeper analysis, integrate with tools like Langfuse or other [tracing backends](../../development/tracing.mdx).
+
+4. **Visualize the pipeline**: Use `pipe.draw()` or `pipe.show()` to see the graph structure and verify your connections are correct. See the [Pipeline Visualization](./visualizing-pipelines.mdx) documentation for details.
+
+5. **Use breakpoints**: Set a `Breakpoint` on a specific component and visit count to inspect the state at that iteration. See [Pipeline Breakpoints](./pipeline-breakpoints.mdx) for details.
+
+6. **Check for blocked pipelines**: If you see a `PipelineComponentsBlockedError`, it means no components can run. This typically indicates a missing connection or a circular dependency. Check that all required inputs are provided.
+
+By combining careful graph design, per-component run limits, and these debugging tools, you can build robust feedback loops in your Haystack pipelines.

docs-website/docs/concepts/pipelines/serialization.mdx

@@ -0,0 +1,276 @@
+---
+title: "Serializing Pipelines"
+id: serialization
+slug: "/serialization"
+description: "Save your pipelines into a custom format and explore the serialization options."
+---
+
+# Serializing Pipelines
+
+Save your pipelines into a custom format and explore the serialization options.
+
+Serialization means converting a pipeline to a format that you can save on your disk and load later.
+
+Haystack supports YAML format for pipeline serialization.
+
+## Converting a Pipeline to YAML
+
+Use the `dumps()` method to convert a Pipeline object to YAML:
+
+```python
+from haystack import Pipeline
+
+pipe = Pipeline()
+print(pipe.dumps())
+
+# Prints:
+#
+# components: {}
+# connections: []
+# max_runs_per_component: 100
+# metadata: {}
+```
+
+You can also use `dump()` method to save the YAML representation of a pipeline in a file:
+
+```python
+with open("/content/test.yml", "w") as file:
+    pipe.dump(file)
+```
+
+## Converting a Pipeline Back to Python
+
+You can convert a YAML pipeline back into Python. Use the `loads()` method to convert a string representation of a pipeline (`str`, `bytes` or `bytearray`)  or the `load()` method to convert a pipeline represented in a file-like object into a corresponding Python object.
+
+Both loading methods support callbacks that let you modify components during the deserialization process. Deserialization is gated by a trusted-module allowlist, so pipelines referencing classes outside of it fail to load until you extend the allowlist — see [Deserialization Security](#deserialization-security) below.
+
+Here is an example script:
+
+```python
+from haystack import Pipeline
+from haystack.core.serialization import DeserializationCallbacks
+from typing import Type, Dict, Any
+
+# This is the YAML you want to convert to Python:
+pipeline_yaml = """
+components:
+  cleaner:
+    init_parameters:
+      remove_empty_lines: true
+      remove_extra_whitespaces: true
+      remove_regex: null
+      remove_repeated_substrings: false
+      remove_substrings: null
+    type: haystack.components.preprocessors.document_cleaner.DocumentCleaner
+  converter:
+    init_parameters:
+      encoding: utf-8
+    type: haystack.components.converters.txt.TextFileToDocument
+connections:
+- receiver: cleaner.documents
+  sender: converter.documents
+max_runs_per_component: 100
+metadata: {}
+"""
+
+
+def component_pre_init_callback(
+    component_name: str,
+    component_cls: Type,
+    init_params: Dict[str, Any],
+):
+    # This function gets called every time a component is deserialized.
+    if component_name == "cleaner":
+        assert "DocumentCleaner" in component_cls.__name__
+        # Modify the init parameters. The modified parameters are passed to
+        # the init method of the component during deserialization.
+        init_params["remove_empty_lines"] = False
+        print("Modified 'remove_empty_lines' to False in 'cleaner' component")
+    else:
+        print(f"Not modifying component {component_name} of class {component_cls}")
+
+
+pipe = Pipeline.loads(
+    pipeline_yaml,
+    callbacks=DeserializationCallbacks(component_pre_init_callback),
+)
+```
+
+## Deserialization Security
+
+Loading a pipeline instantiates the classes referenced in the serialized data. To prevent a crafted YAML file from importing and instantiating arbitrary classes, `Pipeline.load`, `Pipeline.loads`, and `Pipeline.from_dict` refuse to import classes from modules outside a trusted-module allowlist and raise a `DeserializationError` instead.
+
+By default, the allowlist contains `haystack`, `haystack_integrations`, `haystack_experimental`, `builtins`, `typing`, and `collections`. Dangerous builtins such as `eval`, `exec`, `compile`, `__import__`, `open`, and `getattr` are blocked even though `builtins` is allowlisted.
+
+### Allowing Custom Modules
+
+Pipelines that reference custom components or callables in other packages fail to load until you add the modules to the allowlist. You can extend it in three ways:
+
+```python
+from haystack import Pipeline
+
+# 1. Per call: pass additional module patterns for this deserialization only
+pipe = Pipeline.load(open("pipeline.yaml"), allowed_modules=["mypkg.*"])
+
+# 2. Process-wide: extend the allowlist programmatically
+from haystack.core.serialization import allow_deserialization_module
+
+allow_deserialization_module("mypkg")
+```
+
+```shell
+# 3. Environment variable with comma-separated patterns, read on every deserialization call
+export HAYSTACK_DESERIALIZATION_ALLOWLIST="mypkg.*,otherpkg.*"
+```
+
+Patterns are matched as prefixes by default (`"mypkg"` matches `mypkg` and any of its submodules), or as `fnmatch` globs if they contain `*`, `?`, or `[` somewhere other than a trailing `.*`. A trailing `.*` is treated as a prefix match, so `"mypkg"` and `"mypkg.*"` behave identically.
+
+If the source of the serialized data is fully trusted, you can bypass the allowlist entirely with `unsafe=True`:
+
+```python
+pipe = Pipeline.load(open("pipeline.yaml"), unsafe=True)
+```
+
+Only use `unsafe=True` when you fully trust where the serialized pipeline comes from — it also lifts the block on dangerous builtins.
+
+### Nested Init Parameter Validation
+
+As an additional safeguard, deserialization validates the keys of `init_parameters` against the class's `__init__` signature before recursing into any nested `{"type": "...", "init_parameters": {...}}` dictionary. A nested dictionary whose key is not an accepted parameter name is rejected with a `DeserializationError` *before* the nested type is imported, which blocks attempts to smuggle untrusted classes into unused parameter slots. Classes whose constructor takes `**kwargs` are exempt, since their accepted parameter set cannot be statically determined.
+
+This validation may surface pre-existing bugs in YAML files — for example typos, leftovers from renamed or removed parameters, or stale snapshots from older Haystack versions. The fix is to update the YAML so each nested-component key matches a real `__init__` parameter of the parent class.
+
+## Default Serialization Behavior
+
+The serialization system uses `default_to_dict` and `default_from_dict` to handle many object types automatically. You typically do **not** need to implement custom `to_dict`/`from_dict` for:
+
+- **Secrets**: serialized and deserialized automatically so that sensitive values aren't stored in plain text.
+- **ComponentDevice**: device configuration is detected and restored automatically.
+- **Objects with their own `to_dict`/`from_dict`**: any init parameter whose type defines `to_dict()` is serialized by calling it; any dict in `init_parameters` with a `type` key pointing to a class with `from_dict()` is deserialized automatically.
+
+To serialize or deserialize a single component, you can use `component_to_dict` and `component_from_dict` from `haystack.core.serialization`. They use the default behavior above as a fallback when the component doesn't define custom `to_dict`/`from_dict`:
+
+```python
+from haystack import component
+from haystack.core.serialization import component_from_dict, component_to_dict
+
+
+@component
+class Greeter:
+    def __init__(self, message: str = "Hello"):
+        self.message = message
+
+    @component.output_types(greeting=str)
+    def run(self, name: str):
+        return {"greeting": f"{self.message}, {name}!"}
+
+
+# Serialize a component instance to a dictionary
+greeter = Greeter(message="Hi")
+data = component_to_dict(greeter, "my_greeter")
+
+# Deserialize back to a component instance
+restored = component_from_dict(Greeter, data, "my_greeter")
+assert restored.message == greeter.message
+```
+
+:::caution[Init parameters must be stored as instance attributes]
+
+Default serialization only works when there is a **1:1 mapping** between init parameter names and instance attributes. For every argument in `__init__`, the component must assign it to an attribute with the same name. For example, if you have `def __init__(self, prompt: str)`, you must have `self.prompt = prompt` in the class. Otherwise the serialization logic can't find the value to serialize and raises an error or uses the default value if the parameter has one.
+:::
+
+## Performing Custom Serialization
+
+Pipelines and components in Haystack can serialize simple components, including custom ones, out of the box. Code like this just works:
+
+```python
+from haystack import component
+
+
+@component
+class RepeatWordComponent:
+    def __init__(self, times: int):
+        self.times = times
+
+    @component.output_types(result=str)
+    def run(self, word: str):
+        return word * self.times
+```
+
+On the other hand, this code doesn't work if the final format is JSON, as the `set` type is not JSON-serializable:
+
+```python
+from haystack import component
+
+
+@component
+class SetIntersector:
+    def __init__(self, intersect_with: set):
+        self.intersect_with = intersect_with
+
+    @component.output_types(result=set)
+    def run(self, data: set):
+        return data.intersection(self.intersect_with)
+```
+
+In such cases, you can provide your own implementation  `from_dict` and `to_dict` to components:
+
+```python
+from haystack import component, default_from_dict, default_to_dict
+
+
+class SetIntersector:
+    def __init__(self, intersect_with: set):
+        self.intersect_with = intersect_with
+
+    @component.output_types(result=set)
+    def run(self, data: set):
+        return data.intersect(self.intersect_with)
+
+    def to_dict(self):
+        return default_to_dict(self, intersect_with=list(self.intersect_with))
+
+    @classmethod
+    def from_dict(cls, data):
+        # convert the set into a list for the dict representation,
+        # so it can be converted to JSON
+        data["intersect_with"] = set(data["intersect_with"])
+        return default_from_dict(cls, data)
+```
+
+## Saving a Pipeline to a Custom Format
+
+Once a pipeline is available in its dictionary format, the last step of serialization is to convert that dictionary into a format you can store or send over the wire. Haystack supports YAML out of the box, but if you need a different format, you can write a custom Marshaller.
+
+A `Marshaller` is a Python class responsible for converting text to a dictionary and a dictionary to text according to a certain format. Marshallers must respect the `Marshaller` [protocol](https://github.com/deepset-ai/haystack/blob/main/haystack/marshal/protocol.py), providing the methods `marshal` and `unmarshal`.
+
+This is the code for a custom TOML marshaller that relies on the `rtoml` library:
+
+```python
+# This code requires a `pip install rtoml`
+from typing import Dict, Any, Union
+import rtoml
+
+
+class TomlMarshaller:
+    def marshal(self, dict_: Dict[str, Any]) -> str:
+        return rtoml.dumps(dict_)
+
+    def unmarshal(self, data_: Union[str, bytes]) -> Dict[str, Any]:
+        return dict(rtoml.loads(data_))
+```
+
+You can then pass a Marshaller instance to the methods `dump`, `dumps`, `load`, and `loads`:
+
+```python
+from haystack import Pipeline
+from my_custom_marshallers import TomlMarshaller
+
+pipe = Pipeline()
+pipe.dumps(TomlMarshaller())
+# prints:
+# 'max_runs_per_component = 100\nconnections = []\n\n[metadata]\n\n[components]\n'
+```
+
+## Additional References
+
+:notebook: Tutorial: [Serializing LLM Pipelines](https://haystack.deepset.ai/tutorials/29_serializing_pipelines)