chore: import upstream snapshot with attribution

.github/utils/check_imports.py

@@ -0,0 +1,85 @@
+import importlib
+import os
+import sys
+import traceback
+from pathlib import Path
+
+from haystack import logging  # noqa: F401 # this is needed to avoid circular imports
+
+
+def validate_module_imports(root_dir: str, exclude_subdirs: list[str] | None = None) -> tuple[list, list]:
+    """
+    Recursively search for all Python modules and attempt to import them.
+
+    This includes both packages (directories with __init__.py) and individual Python files.
+    """
+    imported = []
+    failed = []
+    exclude_subdirs = (exclude_subdirs or []) + ["__pycache__"]
+
+    # Add the root directory to the Python path
+    sys.path.insert(0, root_dir)
+    base_path = Path(root_dir)
+
+    for root, _, files in os.walk(root_dir):
+        if any(subdir in root for subdir in exclude_subdirs):
+            continue
+
+        # Convert path to module format
+        module_path = ".".join(Path(root).relative_to(base_path.parent).parts)
+        python_files = [f for f in files if f.endswith(".py")]
+
+        # Try importing package and individual files
+        for file in python_files:
+            try:
+                if file == "__init__.py":
+                    module_to_import = module_path
+                else:
+                    module_name = os.path.splitext(file)[0]
+                    module_to_import = f"{module_path}.{module_name}" if module_path else module_name
+
+                importlib.import_module(module_to_import)
+                imported.append(module_to_import)
+            except Exception:
+                failed.append({"module": module_to_import, "traceback": traceback.format_exc()})
+
+    return imported, failed
+
+
+def main() -> None:
+    """
+    This script checks that all Haystack modules can be imported successfully.
+
+    This includes both packages and individual Python files.
+    This can detect several issues, such as:
+    - Syntax errors in Python files
+    - Missing dependencies
+    - Circular imports
+    - Incorrect type hints without forward references
+    """
+    # Add any subdirectories you want to skip during import checks ("__pycache__" is skipped by default)
+    exclude_subdirs = ["testing"]
+
+    print("Checking imports from all Haystack modules...")
+    imported, failed = validate_module_imports(root_dir="haystack", exclude_subdirs=exclude_subdirs)
+
+    if not imported:
+        print("\nNO MODULES WERE IMPORTED")
+        sys.exit(1)
+
+    print(f"\nSUCCESSFULLY IMPORTED {len(imported)} MODULES")
+
+    if failed:
+        print(f"\nFAILED TO IMPORT {len(failed)} MODULES:")
+        for fail in failed:
+            print(f"  - {fail['module']}")
+
+        print("\nERRORS:")
+        for fail in failed:
+            print(f"  - {fail['module']}\n")
+            print(f"    {fail['traceback']}\n\n")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

.github/utils/create_unstable_docs_docusaurus.py

@@ -0,0 +1,102 @@
+"""
+This script creates an unstable documentation version at the time of branch-off for a new Haystack release.
+
+Between branch-off and the actual release, two unstable doc versions coexist.
+If we branch off for 2.20, we have:
+1. the target unstable version, 2.20-unstable (lives in docs-website/versioned_docs/version-2.20-unstable)
+2. the next unstable version, 2.21-unstable (lives in docs-website/docs)
+
+This script takes care of all the necessary updates to the documentation website.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--new-version", help="The new unstable version that is being created (e.g. 2.20).", required=True
+    )
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.new_version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.new_version}"  # e.g., "2.20" - the target release version
+    major, minor = args.new_version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    next_unstable = f"{major}.{int(minor) + 1}-unstable"  # e.g., "2.21-unstable" - next cycle
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    # Check if the versions we're about to create already exist in versioned_docs
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if target_unstable in versions:
+        print(f"{target_unstable} already exists. Nothing to do.")
+        sys.exit(0)
+
+    # Create new unstable from the currently existing one.
+    # The new unstable will be made stable at a later time by another workflow
+    print(f"Creating new unstable version {target_unstable} from main")
+
+    ### Docusaurus updates
+
+    # copy docs to versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/docs", f"docs-website/versioned_docs/version-{target_unstable}")
+
+    # copy reference to reference_versioned_docs/version-target_unstable
+    shutil.copytree("docs-website/reference", f"docs-website/reference_versioned_docs/version-{target_unstable}")
+
+    # generate versioned_sidebars/version-target_unstable-sidebars.json from the current sidebars.js
+    with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+        tmp_path = tmp.name
+    subprocess.run(
+        ["node", "docs-website/scripts/extract_sidebar.mjs", "docs-website/sidebars.js", tmp_path], check=True
+    )
+    docs_sidebar_dest = f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json"
+    shutil.move(tmp_path, docs_sidebar_dest)
+
+    # generate reference_versioned_sidebars/version-target_unstable-sidebars.json from the current reference-sidebars.js
+    ref_sidebar_dest = f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json"
+    with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
+        tmp_path = tmp.name
+    subprocess.run(
+        ["node", "docs-website/scripts/extract_sidebar.mjs", "docs-website/reference-sidebars.js", tmp_path], check=True
+    )
+    shutil.move(tmp_path, ref_sidebar_dest)
+
+    # add unstable version to versions.json
+    with open("docs-website/versions.json") as f:
+        versions_list = json.load(f)
+    versions_list.insert(0, target_unstable)
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # add unstable version to reference_versions.json
+    with open("docs-website/reference_versions.json") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list.insert(0, target_unstable)
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace the target unstable version with the next unstable version
+    with open("docs-website/docusaurus.config.js") as f:
+        config = f.read()
+    config = config.replace(f"label: '{target_unstable}'", f"label: '{next_unstable}'")
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)

.github/utils/docs_search_sync.py

@@ -0,0 +1,120 @@
+"""
+This script syncs the Haystack docs HTML files to the deepset workspace for search indexing.
+
+It is used in the docs_search_sync.yml workflow.
+
+1. Collects all HTML files from the docs and reference directories for the stable Haystack version.
+2. Uploads the HTML files to the deepset workspace.
+    - A timestamp-based metadata field is used to track document versions in the workspace.
+3. Deletes the old HTML files from the deepset workspace.
+    - Since most files are overwritten during upload, only a small number of deletions is expected.
+    - In case MAX_DELETIONS_SAFETY_LIMIT is exceeded, we block the deletion.
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+from deepset_cloud_sdk.workflows.sync_client.files import DeepsetCloudFile, WriteMode, list_files, upload_texts
+
+DEEPSET_WORKSPACE_DOCS_SEARCH = os.environ["DEEPSET_WORKSPACE_DOCS_SEARCH"]
+DEEPSET_API_KEY_DOCS_SEARCH = os.environ["DEEPSET_API_KEY_DOCS_SEARCH"]
+
+# If there are more files to delete than this limit, it's likely that something went wrong in the upload process.
+MAX_DELETIONS_SAFETY_LIMIT = 20
+
+
+def collect_docs_files(version: int) -> list[DeepsetCloudFile]:
+    """
+    Collect all HTML files from the docs and reference directories.
+
+    Returns a list of DeepsetCloudFile objects.
+    """
+    repo_root = Path(__file__).parent.parent.parent
+    build_dir = repo_root / "docs-website" / "build"
+    # we want to exclude previous and temporarily unstable versions (2.x) and next version (next)
+    exclude = ("2.", "next")
+
+    files = []
+    for section in ("docs", "reference"):
+        for subfolder in (build_dir / section).iterdir():
+            if subfolder.is_dir() and not any(x in subfolder.name for x in exclude):
+                for html_file in subfolder.rglob("*.html"):
+                    files.append(
+                        DeepsetCloudFile(
+                            # The build produces files like docs/agents/index.html or reference/agents-api/index.html.
+                            # For file names, we want to use the parent directory name (agents.html or agents-api.html)
+                            name=f"{html_file.parent.name}.html",
+                            text=html_file.read_text(),
+                            meta={
+                                "type": "api-reference" if section == "reference" else "documentation",
+                                "version": version,
+                            },
+                        )
+                    )
+    return files
+
+
+def delete_files(file_names: list[str]) -> None:
+    """
+    Delete files from the deepset workspace.
+    """
+    url = f"https://api.cloud.deepset.ai/api/v1/workspaces/{DEEPSET_WORKSPACE_DOCS_SEARCH}/files"
+    payload = {"names": file_names}
+    headers = {"Accept": "application/json", "Authorization": f"Bearer {DEEPSET_API_KEY_DOCS_SEARCH}"}
+    response = requests.delete(url, json=payload, headers=headers, timeout=300)
+    response.raise_for_status()
+
+
+if __name__ == "__main__":
+    version = time.time_ns()
+    print(f"Docs version: {version}")
+
+    print("Collecting docs files from build directory")
+    dc_files = collect_docs_files(version)
+    print(f"Collected {len(dc_files)} docs files")
+
+    if len(dc_files) == 0:
+        print("No docs files found. Something is wrong. Exiting.")
+        sys.exit(1)
+
+    print("Uploading docs files to deepset")
+    summary = upload_texts(
+        workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH,
+        files=dc_files,
+        api_key=DEEPSET_API_KEY_DOCS_SEARCH,
+        blocking=True,  # Very important to ensure that DC is up to date when we query for deletion
+        timeout_s=300,
+        show_progress=True,
+        write_mode=WriteMode.OVERWRITE,
+        enable_parallel_processing=True,
+    )
+    print(f"Uploaded docs files to deepset\n{summary}")
+    if summary.failed_upload_count > 0:
+        print("Failed to upload some docs files. Stopping to prevent risky deletion of old files.")
+        sys.exit(1)
+
+    print("Listing old docs files from deepset")
+    odata_filter = f"version lt '{version}'"
+    old_files_names = [
+        f.name
+        for batch in list_files(
+            workspace_name=DEEPSET_WORKSPACE_DOCS_SEARCH, api_key=DEEPSET_API_KEY_DOCS_SEARCH, odata_filter=odata_filter
+        )
+        for f in batch
+    ]
+
+    print(f"Found {len(old_files_names)} old files to delete")
+    if len(old_files_names) > MAX_DELETIONS_SAFETY_LIMIT:
+        print(
+            f"Found >{MAX_DELETIONS_SAFETY_LIMIT} old files to delete. "
+            "Stopping because something could have gone wrong in the upload process."
+        )
+        sys.exit(1)
+
+    if len(old_files_names) > 0:
+        print("Deleting old docs files from deepset")
+        delete_files(old_files_names)
+        print("Deleted old docs files from deepset")

.github/utils/docstrings_checksum.py

@@ -0,0 +1,46 @@
+import ast
+import hashlib
+from collections.abc import Iterator
+from pathlib import Path
+
+
+def docstrings_checksum(python_files: Iterator[Path]) -> str:
+    """
+    Calculate the checksum of the docstrings in the given Python files.
+    """
+    files_content = (f.read_text() for f in python_files)
+    trees = (ast.parse(c) for c in files_content)
+
+    # Get all docstrings from async functions, functions,
+    # classes and modules definitions
+    docstrings = []
+    for tree in trees:
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module)):
+                # Skip all node types that can't have docstrings to prevent failures
+                continue
+            docstring = ast.get_docstring(node)
+            if docstring:
+                docstrings.append(docstring)
+
+    # Sort them to be safe, since ast.walk() returns
+    # nodes in no specified order.
+    # See https://docs.python.org/3/library/ast.html#ast.walk
+    docstrings.sort()
+
+    return hashlib.md5(str(docstrings).encode("utf-8")).hexdigest()
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", help="Haystack root folder", required=True, type=Path)
+    args = parser.parse_args()
+
+    # Get all Haystack and rest_api python files
+    root: Path = args.root.absolute()
+    haystack_files = root.glob("haystack/**/*.py")
+
+    md5 = docstrings_checksum(haystack_files)
+    print(md5)

.github/utils/parse_validate_version.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+# parse_validate_version.sh - Parse and validate version for release
+#
+# Usage: ./parse_validate_version.sh <version>
+# Output: Writes to $GITHUB_OUTPUT if set, otherwise to stdout
+#
+# Example:
+#   ./parse_validate_version.sh v2.99.0-rc1
+#
+# This script is used in the release.yml workflow to parse and validate the version to be released.
+# Covers several checks to prevent accidental releases of incorrect versions.
+
+set -euo pipefail
+
+# --- Helpers ---
+
+fail() {
+    echo ""
+    echo -e "❌ $1"
+    echo ""
+    exit 1
+}
+
+ok() {
+    echo "✅ $1"
+}
+
+tag_exists() {
+    git tag -l "$1" | grep -q "^$1$"
+}
+
+branch_exists() {
+    git ls-remote --heads origin "$1" | grep -q "$1"
+}
+
+# --- Parse and validate version ---
+
+VERSION="${1#v}"  # Strip 'v' prefix
+
+echo ""
+echo "ℹ️  Validating: ${1}"
+echo ""
+
+if [[ ! "${VERSION}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-rc([0-9]+))?$ ]]; then
+    fail "Invalid version format: $1\n\n"\
+"Expected format: vMAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH-rcN\n"\
+"Examples: v2.99.0-rc1, v2.99.0, v2.99.1-rc1"
+fi
+ok "Version format is valid"
+
+MAJOR="${BASH_REMATCH[1]}"
+MINOR="${BASH_REMATCH[2]}"
+PATCH="${BASH_REMATCH[3]}"
+RC_NUM="${BASH_REMATCH[5]:-0}"
+
+if [[ "${RC_NUM}" == "0" && "${VERSION}" == *"-rc0" ]]; then
+    fail "Cannot release rc0\n\n"\
+"rc0 is an internal marker created automatically during branch-off.\n"\
+"Release candidates start at rc1."
+fi
+
+MAJOR_MINOR="${MAJOR}.${MINOR}"
+RELEASE_BRANCH="v${MAJOR_MINOR}.x"
+TAG="v${VERSION}"
+
+IS_RC="false"
+[[ "${RC_NUM}" != "0" ]] && IS_RC="true"
+
+IS_FIRST_RC="false"
+if [[ "${PATCH}" == "0" && "${RC_NUM}" == "1" ]]; then
+    IS_FIRST_RC="true"
+fi
+
+# 1. Tag must not already exist
+if tag_exists "${TAG}"; then
+    fail "Version ${TAG} was already released\n\n"\
+"Each version can only be released once.\n"\
+"To publish changes, release the next RC or patch version."
+fi
+ok "Tag ${TAG} does not exist"
+
+# 2. Checks based on release type
+if [[ "${IS_FIRST_RC}" == "true" ]]; then
+    # First RC of minor: branch must NOT exist yet
+    if branch_exists "${RELEASE_BRANCH}"; then
+        fail "Branch ${RELEASE_BRANCH} already exists\n\n"\
+"The first RC of a minor (e.g., v${MAJOR_MINOR}.0-rc1) creates the release branch.\n"\
+"Since the branch exists, this minor was likely already started.\n"\
+"Did you mean to release the next RC (rc2, rc3...) or a patch (v${MAJOR_MINOR}.1-rc1)?"
+    fi
+    ok "Branch ${RELEASE_BRANCH} does not exist"
+
+    # First RC of minor: VERSION.txt must contain rc0
+    EXPECTED="${MAJOR_MINOR}.0-rc0"
+    ACTUAL=$(cat VERSION.txt)
+    if [[ "${ACTUAL}" != "${EXPECTED}" ]]; then
+        ACTUAL_MINOR=$(echo "${ACTUAL}" | cut -d. -f1,2)
+        fail "Cannot release v${MAJOR_MINOR}.0-rc1 from this branch\n\n"\
+"The main branch is prepared for version ${ACTUAL_MINOR}, not ${MAJOR_MINOR}.\n"\
+"Check that you're releasing the correct version."
+    fi
+    ok "VERSION.txt = ${EXPECTED}"
+
+else
+    # Not first RC: branch MUST exist
+    if ! branch_exists "${RELEASE_BRANCH}"; then
+        if [[ "${PATCH}" == "0" ]]; then
+            fail "Branch ${RELEASE_BRANCH} does not exist\n\n"\
+"For subsequent RCs (rc2, rc3...), the release branch must already exist.\n"\
+"Release the first RC (v${MAJOR_MINOR}.0-rc1) first to create the branch."
+        else
+            fail "Branch ${RELEASE_BRANCH} does not exist\n\n"\
+"For patch releases, the release branch must already exist.\n"\
+"The minor version (v${MAJOR_MINOR}.0) must be released before any patches."
+        fi
+    fi
+    ok "Branch ${RELEASE_BRANCH} exists"
+
+    # Subsequent RC (rc2, rc3...): previous RC must exist
+    if [[ "${RC_NUM}" -gt 1 ]]; then
+        PREV_RC_NUM=$((RC_NUM - 1))
+        PREV_TAG="v${MAJOR_MINOR}.${PATCH}-rc${PREV_RC_NUM}"
+        if ! tag_exists "${PREV_TAG}"; then
+            fail "Cannot release v${MAJOR_MINOR}.${PATCH}-rc${RC_NUM}\n\n"\
+"Previous RC (${PREV_TAG}) was not found.\n"\
+"RC versions must be sequential. Release rc${PREV_RC_NUM} first."
+        fi
+        ok "Previous tag ${PREV_TAG} exists"
+    fi
+
+    # Final release: at least one RC must exist
+    if [[ "${RC_NUM}" == "0" ]]; then
+        RC_TAGS=$(git tag -l "v${MAJOR_MINOR}.${PATCH}-rc*" | grep -v "\-rc0$" || true)
+        if [[ -z "${RC_TAGS}" ]]; then
+            fail "Cannot release stable version v${MAJOR_MINOR}.${PATCH}\n\n"\
+"No release candidate found for this version.\n"\
+"Stable releases require at least one RC first (e.g., v${MAJOR_MINOR}.${PATCH}-rc1)."
+        fi
+        LAST_RC=$(echo "${RC_TAGS}" | sort -V | tail -n1)
+        ok "Found RC: ${LAST_RC}"
+
+        # Check Tests workflow passed (only if credentials available)
+        if [[ -n "${GH_TOKEN:-}" && -n "${GITHUB_REPOSITORY:-}" ]]; then
+            RC_SHA=$(git rev-list -n 1 "${LAST_RC}")
+            RESULT=$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${RC_SHA}&status=success" \
+                --jq '.workflow_runs[] | select(.name == "Tests") | .conclusion' 2>/dev/null || true)
+            if [[ -z "${RESULT}" ]]; then
+                fail "Cannot release stable version v${MAJOR_MINOR}.${PATCH}\n\n"\
+"Tests did not pass on the last RC (${LAST_RC}).\n"\
+"Wait for tests to complete, or release a new RC with fixes."
+            fi
+            ok "Tests passed on ${LAST_RC}"
+        fi
+    fi
+fi
+
+echo ""
+ok "All validations passed!"
+echo ""
+
+# --- Output to GITHUB_OUTPUT (or stdout for local testing) ---
+
+OUTPUT_FILE="${GITHUB_OUTPUT:-/dev/stdout}"
+
+{
+    echo "version=${VERSION}"
+    echo "major_minor=${MAJOR_MINOR}"
+    echo "release_branch=${RELEASE_BRANCH}"
+    echo "is_rc=${IS_RC}"
+    echo "is_first_rc=${IS_FIRST_RC}"
+} >> "${OUTPUT_FILE}"

.github/utils/prepare_release_notification.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# prepare_release_notification.sh - Prepare Slack notification for release outcome
+#
+# Requires: VERSION, RUN_URL, HAS_FAILURE, GH_TOKEN, GITHUB_REPOSITORY
+# Optional: IS_RC, IS_FIRST_RC, MAJOR_MINOR, GITHUB_URL, PYPI_URL, DOCKER_URL,
+#           BUMP_VERSION_PR_URL, DC_PIPELINE_TEMPLATES_PR_URL, DC_CUSTOM_NODES_PR_URL,
+#           HAYSTACK_RUNTIME_PR_URL, GITHUB_WORKSPACE
+# Output: slack_payload.json
+#
+# This script is used in the release.yml workflow to prepare the notification payload
+# sent to Slack after a release completes (success or failure).
+# Text uses Slack mrkdwn format: *bold*, <url|label> for links.
+
+set -euo pipefail
+
+PAYLOAD_FILE="${GITHUB_WORKSPACE:-/tmp}/slack_payload.json"
+
+write_payload() {
+  jq -n --arg text "$TXT" '{
+    text: $text,
+    blocks: [{ type: "section", text: { type: "mrkdwn", text: $text } }]
+  }' > "$PAYLOAD_FILE"
+}
+
+if [[ "${HAS_FAILURE}" == "true" ]]; then
+  TXT=":red_circle: Release *${VERSION}* failed"
+  TXT+=$'\n'"Check workflow run for details: <${RUN_URL}|View Logs>"
+  write_payload
+  exit 0
+fi
+
+# Success case
+
+TXT=":white_check_mark: Release *${VERSION}* completed successfully"
+
+# Add artifact URLs if available
+if [[ -n "${GITHUB_URL:-}" || -n "${PYPI_URL:-}" || -n "${DOCKER_URL:-}" ]]; then
+  TXT+=$'\n\n:package: *Artifacts:*'
+  [[ -n "${GITHUB_URL:-}" ]] && TXT+=$'\n'"- <${GITHUB_URL}|Release notes (GitHub)>"
+  [[ -n "${PYPI_URL:-}" ]] && TXT+=$'\n'"- <${PYPI_URL}|PyPI>"
+  [[ -n "${DOCKER_URL:-}" ]] && TXT+=$'\n'"- <${DOCKER_URL}|Docker>"
+fi
+
+# For RCs, include link to the Tests workflow run
+if [[ "${IS_RC:-}" == "true" ]]; then
+  COMMIT_SHA=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${VERSION}" --jq '.sha' 2>/dev/null || echo "")
+  if [[ -n "${COMMIT_SHA}" ]]; then
+    TESTS_RUN=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${COMMIT_SHA}" \
+      --jq '.workflow_runs[] | select(.name == "Tests") | .html_url' 2>/dev/null | head -1 || echo "")
+    if [[ -n "${TESTS_RUN}" ]]; then
+      TXT+=$'\n\n'":test_tube: <${TESTS_RUN}|Haystack Tests>"
+    fi
+  fi
+fi
+
+# For first RC, include the PRs to merge from branch-off
+if [[ "${IS_FIRST_RC:-}" == "true" && -n "${BUMP_VERSION_PR_URL:-}" ]]; then
+  TXT+=$'\n\n'":clipboard: *PRs to merge:*"
+  TXT+=$'\n'"- <${BUMP_VERSION_PR_URL}|Bump unstable version and create unstable docs>"
+fi
+
+# For RCs, include Platform test PRs
+if [[ "${IS_RC:-}" == "true" ]]; then
+  PLATFORM_PRS=""
+  [[ -n "${DC_PIPELINE_TEMPLATES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_PIPELINE_TEMPLATES_PR_URL}|dc-pipeline-templates>"
+  [[ -n "${DC_CUSTOM_NODES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_CUSTOM_NODES_PR_URL}|deepset-cloud-custom-nodes>"
+  [[ -n "${HAYSTACK_RUNTIME_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${HAYSTACK_RUNTIME_PR_URL}|haystack-runtime>"
+  if [[ -n "${PLATFORM_PRS}" ]]; then
+    TXT+=$'\n\n'":factory: *Test PRs opened on Platform:*${PLATFORM_PRS}"
+  fi
+fi
+
+# For RCs, request Platform Engineering taking over testing
+if [[ "${IS_RC:-}" == "true" ]]; then
+  TXT+=$'\n\n'"This release is marked as a Release Candidate."
+  TXT+=$'\n'"Notify #deepset-platform-engineering channel on Slack that the RC is available"
+  TXT+=" and that Platform tests pass/fail, linking the PRs opened on the Platform repositories."
+fi
+
+# For final minor releases (vX.Y.0), include the docs promotion PR
+if [[ "${VERSION}" =~ ^v[0-9]+\.[0-9]+\.0$ && -n "${MAJOR_MINOR:-}" ]]; then
+  PROMOTE_DOCS_PR_URL=$(gh pr list --repo "${GITHUB_REPOSITORY}" \
+    --head "promote-unstable-docs-${MAJOR_MINOR}" --json url --jq '.[0].url' 2>/dev/null || echo "")
+  if [[ -n "${PROMOTE_DOCS_PR_URL}" ]]; then
+    TXT+=$'\n\n'":clipboard: *PRs to merge:*"
+    TXT+=$'\n'"- <${PROMOTE_DOCS_PR_URL}|Promote unstable docs>"
+  fi
+fi
+
+# For final releases (not RCs), include info about pushing release notes to website
+if [[ "${IS_RC:-}" != "true" ]]; then
+  TXT+=$'\n\n'":memo: After refining and finalizing release notes, push them to Haystack website:"
+  TXT+=$'\n'"\`gh workflow run push_release_notes_to_website.yml -R deepset-ai/haystack -f version=${VERSION}\`"
+fi
+
+write_payload

.github/utils/promote_unstable_docs_docusaurus.py

@@ -0,0 +1,117 @@
+"""
+This script promotes an unstable documentation version to a stable version at the time of a new Haystack release.
+
+To understand how unstable doc versions are created, see create_unstable_docs_docusaurus.py.
+"""
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+MAX_STABLE_VERSIONS = 5
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version", help="The version to promote to stable (e.g. 2.20).", required=True)
+    args = parser.parse_args()
+
+    if VERSION_VALIDATOR.match(args.version) is None:
+        sys.exit("Version must be formatted like so <major>.<minor>")
+
+    target_version = f"{args.version}"  # e.g., "2.20" - the target release version
+    major, minor = args.version.split(".")
+
+    target_unstable = f"{target_version}-unstable"  # e.g., "2.20-unstable"
+    previous_stable = f"{major}.{int(minor) - 1}"  # e.g., "2.19" - previous stable release
+
+    versions = [
+        folder.replace("version-", "")
+        for folder in os.listdir("docs-website/versioned_docs")
+        if os.path.isdir(os.path.join("docs-website/versioned_docs", folder))
+    ]
+
+    if target_version in versions:
+        sys.exit(f"{target_version} already exists (already released). Aborting.")
+    if target_unstable not in versions:
+        sys.exit(f"Can't find version {target_unstable} to promote to {target_version}")
+
+    print(f"Promoting unstable version {target_unstable} to stable version {target_version}")
+
+    ### Docusaurus updates
+
+    # move versioned_docs/version-target_unstable to versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/versioned_docs/version-{target_unstable}",
+        f"docs-website/versioned_docs/version-{target_version}",
+    )
+
+    # move reference_versioned_docs/version-target_unstable to reference_versioned_docs/version-target_version
+    shutil.move(
+        f"docs-website/reference_versioned_docs/version-{target_unstable}",
+        f"docs-website/reference_versioned_docs/version-{target_version}",
+    )
+
+    # move versioned_sidebars/version-target_unstable-sidebars.json
+    # to versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # move reference_versioned_sidebars/version-target_unstable-sidebars.json
+    # to reference_versioned_sidebars/version-target_version-sidebars.json
+    shutil.move(
+        f"docs-website/reference_versioned_sidebars/version-{target_unstable}-sidebars.json",
+        f"docs-website/reference_versioned_sidebars/version-{target_version}-sidebars.json",
+    )
+
+    # replace unstable version with stable version in versions.json
+    with open("docs-website/versions.json") as f:
+        versions_list = json.load(f)
+    versions_list[versions_list.index(target_unstable)] = target_version
+    with open("docs-website/versions.json", "w") as f:
+        json.dump(versions_list, f)
+
+    # replace unstable version with stable version in reference_versions.json
+    with open("docs-website/reference_versions.json") as f:
+        reference_versions_list = json.load(f)
+    reference_versions_list[reference_versions_list.index(target_unstable)] = target_version
+    with open("docs-website/reference_versions.json", "w") as f:
+        json.dump(reference_versions_list, f)
+
+    # in docusaurus.config.js, replace previous stable version with the target version
+    with open("docs-website/docusaurus.config.js") as f:
+        config = f.read()
+    config = config.replace(f"lastVersion: '{previous_stable}'", f"lastVersion: '{target_version}'")  # "2.19" -> "2.20"
+    with open("docs-website/docusaurus.config.js", "w") as f:
+        f.write(config)
+
+    # regenerate vercel.json redirects for inactive versions (those beyond the top MAX_STABLE_VERSIONS)
+    with open("docs-website/versions.json") as f:
+        updated_versions = json.load(f)
+    stable_versions = [v for v in updated_versions if not v.endswith("-unstable")]
+    inactive_versions = stable_versions[MAX_STABLE_VERSIONS:]
+    redirects = []
+    for v in inactive_versions:
+        redirects.append({"source": f"/docs/{v}/:slug*", "destination": "/docs/:slug*", "permanent": True})
+        redirects.append({"source": f"/reference/{v}/:slug*", "destination": "/reference/:slug*", "permanent": True})
+
+    with open("docs-website/vercel.json") as f:
+        vercel_config = json.load(f)
+
+    existing_redirects = vercel_config.get("redirects", [])
+    existing_sources = {r.get("source") for r in existing_redirects}
+
+    for r in redirects:
+        if r["source"] not in existing_sources:
+            existing_redirects.append(r)
+
+    vercel_config["redirects"] = existing_redirects
+    with open("docs-website/vercel.json", "w") as f:
+        json.dump(vercel_config, f, indent=2)
+        f.write("\n")
+    print(f"Updated vercel.json with {len(redirects)} redirect(s) for inactive versions: {inactive_versions}")

.github/utils/pyproject_to_requirements.py

@@ -0,0 +1,53 @@
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import toml
+
+matcher = re.compile(r"farm-haystack\[(.+)\]")
+parser = argparse.ArgumentParser(
+    prog="pyproject_to_requirements.py", description="Convert pyproject.toml to requirements.txt"
+)
+parser.add_argument("pyproject_path")
+parser.add_argument("--extra", default="")
+
+
+def resolve(target: str, extras: dict, results: set) -> None:
+    """
+    Resolve the dependencies for a given target.
+    """
+    if target not in extras:
+        results.add(target)
+        return
+
+    for t in extras[target]:
+        m = matcher.match(t)
+        if m:
+            for i in m.group(1).split(","):
+                resolve(i, extras, results)
+        else:
+            resolve(t, extras, results)
+
+
+def main(pyproject_path: Path, extra: str = "") -> None:
+    """
+    Convert a pyproject.toml file to a requirements.txt file.
+    """
+    content = toml.load(pyproject_path)
+    # basic set of dependencies
+    deps = set(content["project"]["dependencies"])
+
+    if extra:
+        extras = content["project"]["optional-dependencies"]
+        resolve(extra, extras, deps)
+
+    sys.stdout.write("\n".join(sorted(deps)))
+    sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    pyproject_path = Path(args.pyproject_path).absolute()
+
+    main(pyproject_path, args.extra)

.github/utils/update_haystack_dc_custom_nodes.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Update the haystack-ai version in the deepset-cloud-custom-nodes uv.lock file.
+
+Fetches sdist/wheel hashes from PyPI and updates the haystack-ai package entry,
+preserving the existing lock file formatting.
+"""
+
+import argparse
+import json
+import sys
+import urllib.request
+
+import tomlkit
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version", help="Version to update to (e.g. 2.26.1-rc1)")
+    parser.add_argument("lock_file", help="Path to uv.lock")
+    args = parser.parse_args()
+
+    # PEP 440 normalized version for filenames
+    new_version = args.version.replace("-", "")
+
+    # Fetch hashes from PyPI
+    pypi_data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/haystack-ai/{args.version}/json"))
+    sdist_sha = wheel_sha = None
+    for u in pypi_data["urls"]:
+        if u["packagetype"] == "sdist":
+            sdist_sha = u["digests"]["sha256"]
+        elif u["packagetype"] == "bdist_wheel":
+            wheel_sha = u["digests"]["sha256"]
+    if not sdist_sha or not wheel_sha:
+        sys.exit("Could not find sdist or wheel hashes on PyPI")
+
+    with open(args.lock_file) as f:
+        data = tomlkit.load(f)
+
+    found = False
+    for pkg in data["package"]:
+        if pkg["name"] == "haystack-ai":
+            old_version = pkg["version"]
+
+            pkg["version"] = new_version
+
+            pkg["sdist"]["url"] = pkg["sdist"]["url"].replace(old_version, new_version)
+            pkg["sdist"]["hash"] = f"sha256:{sdist_sha}"
+
+            wheel = pkg["wheels"][0]
+            wheel["url"] = wheel["url"].replace(old_version, new_version)
+            wheel["hash"] = f"sha256:{wheel_sha}"
+
+            found = True
+            print(f"Updated haystack-ai from {old_version} to {new_version}")
+            break
+
+    if not found:
+        sys.exit("haystack-ai package not found in uv.lock")
+
+    with open(args.lock_file, "w") as f:
+        tomlkit.dump(data, f)

.github/utils/wait_for_workflows.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# wait_for_workflows.sh - Wait for tag-triggered workflows to complete
+#
+# Usage: ./wait_for_workflows.sh <tag> <workflow_name1> [workflow_name2] ...
+# Requires: GH_TOKEN and GITHUB_REPOSITORY environment variables
+#
+# Example:
+#   ./wait_for_workflows.sh v2.19.0 "Project release on PyPi" "Docker image release"
+#
+# This script is used in the release.yml workflow to wait for the workflows triggered by a specific release tag to
+# successfully complete.
+
+
+# With the default values, we wait for 20 minutes
+MAX_ATTEMPTS="${MAX_ATTEMPTS:-40}"
+SLEEP_SECONDS="${SLEEP_SECONDS:-30}"
+
+set -euo pipefail
+
+if [[ -z "${GH_TOKEN:-}" ]] || [[ -z "${GITHUB_REPOSITORY:-}" ]]; then
+    echo "❌ GH_TOKEN and GITHUB_REPOSITORY must be set"
+    exit 1
+fi
+
+TAG="$1"
+shift
+WORKFLOWS=("$@")
+
+
+# Get commit SHA from tag
+TAG_SHA=$(git rev-list -n 1 "${TAG}" 2>/dev/null) || {
+    echo "❌ Tag ${TAG} not found"
+    exit 1
+}
+
+echo "Tag ${TAG} (commit: ${TAG_SHA:0:7})"
+echo ""
+
+wait_for_workflow() {
+    local name="$1"
+    echo "⏳ Waiting for: $name"
+
+    for ((i=1; i<=MAX_ATTEMPTS; i++)); do
+        jq_filter="[.workflow_runs[] | select(.head_sha == \"${TAG_SHA}\" and .name == \"${name}\")]
+            | sort_by(.created_at) | last"
+
+        result=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs" \
+            --jq "$jq_filter" 2>/dev/null || echo "")
+
+        if [[ -z "$result" ]]; then
+            echo "   Attempt $i/$MAX_ATTEMPTS: not started yet..."
+            sleep $SLEEP_SECONDS
+            continue
+        fi
+
+        status=$(echo "$result" | jq -r '.status')
+        conclusion=$(echo "$result" | jq -r '.conclusion')
+
+        if [[ "$status" == "completed" ]]; then
+            if [[ "$conclusion" == "success" ]]; then
+                echo "✅ $name completed"
+                return 0
+            else
+                echo "❌ $name failed: $conclusion"
+                return 1
+            fi
+        fi
+
+        echo "   Attempt $i/$MAX_ATTEMPTS: $status..."
+        sleep $SLEEP_SECONDS
+    done
+
+    echo "❌ $name: timeout after $((MAX_ATTEMPTS * SLEEP_SECONDS / 60)) minutes"
+    return 1
+}
+
+for workflow in "${WORKFLOWS[@]}"; do
+    wait_for_workflow "$workflow" || exit 1
+done
+
+echo ""
+echo "✅ All workflows completed"