chore: import upstream snapshot with attribution

.github/workflows/auto_approve_api_ref_sync.yml

@@ -0,0 +1,35 @@
+name: Approve and merge API reference sync PRs
+
+# Automatically approve and merge API reference sync PRs from Haystack, Haystack Core Integrations,
+# and Haystack Experimental
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "docs-website/reference/**"
+      - "docs-website/reference_versioned_docs/**"
+
+permissions:
+  contents: read
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  auto-approve-and-merge:
+    permissions:
+      contents: write
+      pull-requests: write
+    if: |
+      github.event.pull_request.user.login == 'HaystackBot' &&
+      startsWith(github.event.pull_request.head.ref, 'sync-docusaurus-api-reference') &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-slim
+    steps:
+      - name: Approve PR
+        run: gh pr review --approve ${{ github.event.pull_request.number }} --repo ${{ github.repository }}
+
+      - name: Enable auto-merge
+        run: gh pr merge --squash --auto ${{ github.event.pull_request.number }} --repo ${{ github.repository }}

.github/workflows/branch_off.yml

@@ -0,0 +1,99 @@
+name: Branch off
+
+on:
+  workflow_call:
+    outputs:
+      bump_version_pr_url:
+        description: 'URL of the bump version PR'
+        value: ${{ jobs.branch-off.outputs.bump_version_pr_url }}
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  branch-off:
+    runs-on: ubuntu-slim
+    outputs:
+      bump_version_pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: main
+          # Persist the bot token so the branch/tag pushes below authenticate as
+          # HaystackBot (a ruleset-bypass actor) instead of the default
+          # github-actions[bot], which only has contents:read and is blocked by
+          # the release rulesets.
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Define all versions
+        id: versions
+        shell: bash
+        run: |
+          # example: 2.20.0-rc0 in VERSION.txt -> 2.20
+          echo "current_version_major_minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"
+          # example: 2.20.0-rc0 in VERSION.txt -> 2.21.0-rc0
+          echo "next_version_rc0=$(awk -F. '/[0-9]+\./{$2++;print}' OFS=. < VERSION.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Create release branch and tag
+        shell: bash
+        env:
+          # We use the HAYSTACK_BOT_TOKEN here so the PR created by the step will
+          # trigger required workflows and can be merged by anyone
+          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create the release branch from main
+          git checkout -b v${{ steps.versions.outputs.current_version_major_minor }}.x
+          git push -u origin v${{ steps.versions.outputs.current_version_major_minor }}.x
+
+          # Tag the branch-off point with the next version rc0 to mark start of next dev cycle.
+          # The tag points to this commit (before VERSION.txt is bumped).
+          # This is intentional for reno to work properly.
+          git tag "v${{ steps.versions.outputs.next_version_rc0 }}" -m "v${{ steps.versions.outputs.next_version_rc0 }}"
+          git push --tags
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Prepare changes for main
+        shell: bash
+        run: |
+          git checkout main
+
+          # Bump VERSION.txt to next version rc0
+          echo "${{ steps.versions.outputs.next_version_rc0 }}" > VERSION.txt
+
+          # Generate unstable docs for Docusaurus
+          python ./.github/utils/create_unstable_docs_docusaurus.py --new-version ${{ steps.versions.outputs.current_version_major_minor }}
+
+      - name: Create PR to bump unstable version and create unstable docs
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Bump unstable version and create unstable docs"
+          branch: bump-version
+          base: main
+          title: "Bump unstable version and create unstable docs"
+          add-paths: |
+            VERSION.txt
+            docs-website
+          body: |
+            This PR:
+            - Bumps the unstable version to `${{ steps.versions.outputs.next_version_rc0 }}`
+            - Creates the unstable docs for Haystack ${{ steps.versions.outputs.current_version_major_minor }}
+
+            You can inspect the docs preview (two unstable versions will be available) and merge it.
+          labels: "ignore-for-release-notes"
+          reviewers: "${{ github.actor }}"

.github/workflows/cflite_pr.yml

@@ -0,0 +1,57 @@
+name: ClusterFuzzLite PR fuzzing
+
+# Short, code-change-scoped fuzzing run on PRs that touch fuzzed code or the
+# fuzzing setup. Catches regressions and crashes introduced by a change.
+# Continuous/batch fuzzing can be added later as a separate scheduled workflow.
+#
+# Scoped to the modules the harnesses actually exercise rather than all of
+# `haystack/**`, so the (build-heavy) job doesn't run on every library PR:
+#   - Pipeline.loads          -> core/pipeline, core/serialization, marshal
+#   - Document.from_dict       -> dataclasses
+#   - document_matches_filter  -> utils/filters
+# Deep, transitively-reached regressions are better caught by a scheduled batch
+# run; random YAML/JSON essentially never reaches haystack/components.
+on:
+  pull_request:
+    paths:
+      - "haystack/core/pipeline/**"
+      - "haystack/core/serialization.py"
+      - "haystack/core/errors.py"
+      - "haystack/marshal/**"
+      - "haystack/dataclasses/**"
+      - "haystack/utils/filters.py"
+      - "test/fuzz/**"
+      - ".clusterfuzzlite/**"
+      - ".github/workflows/cflite_pr.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  pr-fuzzing:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: cflite-pr-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    steps:
+      - name: Build Fuzzers
+        id: build
+        uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+        with:
+          language: python
+          sanitizer: address
+          # Needed so build_fuzzers can check out the PR base for `mode: code-change`
+          # diffing in the Run Fuzzers step below.
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Fuzzers
+        id: run
+        uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fuzz-seconds: 120
+          mode: code-change
+          sanitizer: address
+          # Crashes fail the job and are uploaded as artifacts. SARIF upload is
+          # disabled to keep the token least-privilege (no security-events: write).
+          output-sarif: false

.github/workflows/check_api_ref.yml

@@ -0,0 +1,99 @@
+name: Check API reference changes
+
+on:
+  pull_request:
+    paths:
+      - "haystack/**/*.py"
+      - "pydoc/*.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  test-api-reference-build:
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.13"
+
+      - name: Detect API reference changes
+        id: changed
+        shell: python
+        run: |
+          import os
+          import subprocess
+          from pathlib import Path
+
+          import sys
+          sys.path.insert(0, ".github/utils")
+          from docstrings_checksum import docstrings_checksum
+
+          def git(*args):
+              result = subprocess.run(["git", *args], capture_output=True, text=True)
+              return result.stdout.strip(), result.returncode
+
+          base_sha, _ = git("rev-parse", "HEAD^1")
+          diff_output, _ = git("diff", "--name-only", f"{base_sha}...HEAD")
+          changed_files = set(diff_output.splitlines())
+
+          needs_check = False
+
+          # If any pydoc config changed, always rebuild
+          if any(f.startswith("pydoc/") and f.endswith(".yml") for f in changed_files):
+              needs_check = True
+
+          # If Python files changed, compare docstring checksums
+          if not needs_check and any(f.startswith("haystack/") and f.endswith(".py") for f in changed_files):
+              runner_temp = os.environ["RUNNER_TEMP"]
+              base_worktree = os.path.join(runner_temp, "base")
+              _, rc = git("worktree", "add", base_worktree, base_sha)
+
+              pr_checksum = docstrings_checksum(Path(".").glob("haystack/**/*.py"))
+              base_checksum = ""
+              if rc == 0:
+                  base_checksum = docstrings_checksum(Path(base_worktree).glob("haystack/**/*.py"))
+
+              if pr_checksum != base_checksum:
+                  needs_check = True
+
+          print(f"API reference check needed: {needs_check}")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+              f.write(f"needs_check={str(needs_check).lower()}\n")
+
+      - name: Install Hatch
+        if: steps.changed.outputs.needs_check == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch --uploaded-prior-to=P1D
+
+      - name: Generate API references
+        if: steps.changed.outputs.needs_check == 'true'
+        run: hatch run docs
+
+      - name: Set up Node.js
+        if: steps.changed.outputs.needs_check == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Run Docusaurus md/mdx checker
+        if: steps.changed.outputs.needs_check == 'true'
+        working-directory: tmp_api_reference
+        run: |
+          # docusaurus-mdx-checker is a package that is not frequently updated. Its dependency katex sometimes ships a
+          # broken ESM build, where a __VERSION__ placeholder is left unresolved, causing a ReferenceError at import time.
+          # Node 22+ prefers ESM when available. We force CJS (CommonJS) resolution to use the working katex build.
+          # This should be safe because docusaurus-mdx-checker and its dependencies provide CJS builds.
+          export NODE_OPTIONS="--conditions=require"
+          npx docusaurus-mdx-checker@3.0.0 -v || {
+              echo ""
+              echo "For common MDX problems, see https://docusaurus.io/blog/preparing-your-site-for-docusaurus-v3#common-mdx-problems"
+              exit 1
+            }

.github/workflows/codeql.yml

@@ -0,0 +1,56 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Cancel a superseded run for the same ref (e.g. a force-push to a PR) so we
+# don't burn CI minutes analyzing commits that are already stale.
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      # Required to upload code-scanning results to the Security tab.
+      security-events: write
+      # Required by codeql-action to read the workflow/CI config.
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Haystack is a Python project; Python is interpreted, so no build is
+          # needed. Add `javascript-typescript` here if the docs-website JS code
+          # should also be scanned.
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/coverage_comment.yml

@@ -0,0 +1,24 @@
+name: Add comment about test coverage to PRs
+
+on:
+  workflow_run:
+    workflows:
+      - "Tests"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  comment:
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: py-cov-action/python-coverage-comment-action@5d8df5979747514c914e1c5a12335a7cf9a2745f # v4.1
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 60

.github/workflows/docker_release.yml

@@ -0,0 +1,88 @@
+name: Docker image release
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/docker_release.yml'
+      - 'docker/**'
+      - 'haystack/**'
+      - 'pyproject.toml'
+      - 'VERSION.txt'
+    tags:
+      - "v2.[0-9]+.[0-9]+*"
+
+env:
+  DOCKER_REPO_NAME: deepset/haystack
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    name: Build base image
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepset-ai'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      - name: Login to DockerHub
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        with:
+          images: $DOCKER_REPO_NAME
+
+      - name: Detect stable version
+        run: |
+          if [[ "${{ steps.meta.outputs.version }}" =~ ^v2\.[0-9]+\.[0-9]+$ ]]; then
+            echo "IS_STABLE=true" >> "$GITHUB_ENV"
+            echo "Stable version detected"
+          else
+            echo "Not a stable version"
+          fi
+      - name: Build base images
+        uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0
+        env:
+          IMAGE_TAG_SUFFIX: ${{ steps.meta.outputs.version }}
+          HAYSTACK_VERSION: ${{ steps.meta.outputs.version }}
+        with:
+          source: ./docker
+          targets: base
+          push: true
+
+      - name: Test base image
+        run: |
+          EXPECTED_VERSION=$(cat VERSION.txt)
+          if [[ $EXPECTED_VERSION == *"-"* ]]; then
+            EXPECTED_VERSION=$(cut -d '-' -f 1 < VERSION.txt)$(cut -d '-' -f 2 < VERSION.txt)
+          fi
+          TAG="base-${{ steps.meta.outputs.version }}"
+
+          PLATFORM="linux/amd64"
+          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
+          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"
+
+          PLATFORM="linux/arm64"
+          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
+          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"
+
+          # Remove image after test to avoid filling the GitHub runner and prevent its failure
+          docker rmi "deepset/haystack:$TAG"

.github/workflows/docs-website-test-docs-snippets.yml

@@ -0,0 +1,74 @@
+name: Test Python snippets in docs
+
+on:
+  schedule:
+    - cron: "17 3 * * *" # daily at 03:17 UTC
+  workflow_dispatch:
+    inputs:
+      haystack_version:
+        description: "Haystack version to test against (e.g., 2.16.1, main)"
+        required: false
+        default: "main"
+        type: string
+
+env:
+  HATCH_VERSION: "1.16.5"
+  PYTHON_VERSION: "3.11"
+
+permissions:
+  contents: read
+
+jobs:
+  test-docs-snippets:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+      AZURE_OPENAI_AD_TOKEN: ${{ secrets.AZURE_OPENAI_AD_TOKEN }}
+      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+      CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
+      HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: sudo apt-get install -y ffmpeg
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Generate API reference for Docusaurus
+        run: hatch run docs
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests toml --uploaded-prior-to=P1D
+
+      - name: Run snippet tests (verbose)
+        shell: bash
+        run: |
+          hatch -e test env run -- python docs-website/scripts/test_python_snippets.py --verbose tmp_api_reference/
+
+  notify-slack-on-failure:
+    if: failure() && github.ref_name == 'main'
+    needs:
+      - test-docs-snippets
+    runs-on: ubuntu-slim
+    steps:
+      - uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/docs_search_sync.yml

@@ -0,0 +1,60 @@
+name: Docs Search Sync
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 1 * * *"
+
+# Running this workflow multiple times in parallel can cause issues with files uploads and deletions.
+concurrency:
+  group: docs-search-sync
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  docs-search-sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Install Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Install Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Docusaurus and build docs-website
+        working-directory: docs-website
+        run: |
+          npm install --ignore-scripts
+          npm rebuild sharp
+          npm run build
+
+      - name: Install script dependencies
+        # sniffio is needed because of https://github.com/deepset-ai/deepset-cloud-sdk/issues/286
+        # we pin pyrate-limiter due to https://github.com/deepset-ai/deepset-cloud-sdk/issues/295
+        run: |
+          python -m pip install --upgrade pip
+          pip install deepset-cloud-sdk sniffio requests "pyrate-limiter<4" --uploaded-prior-to=P1D
+
+      - name: Update new docs to Search pipeline and remove outdated docs
+        env:
+          DEEPSET_WORKSPACE_DOCS_SEARCH: ${{ secrets.DEEPSET_WORKSPACE_DOCS_SEARCH }}
+          DEEPSET_API_KEY_DOCS_SEARCH: ${{ secrets.DEEPSET_API_KEY_DOCS_SEARCH }}
+        run: python ./.github/utils/docs_search_sync.py
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/docstring_labeler.yml

@@ -0,0 +1,70 @@
+name: Detect docstrings edit
+
+# This workflow runs in the *untrusted* pull_request context: it has a read-only
+# token and no access to secrets, so checking out the PR head here is safe.
+# It only computes whether docstrings changed and hands the result to the
+# "Apply docstrings label" workflow (workflow_run), which holds the write token
+# but never checks out untrusted code. See:
+# https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
+on:
+  pull_request:
+    paths:
+      - "haystack/**/*.py"
+
+permissions:
+  contents: read
+
+env:
+  PYTHON_VERSION: "3.11"
+
+jobs:
+  detect:
+    runs-on: ubuntu-slim
+
+    steps:
+      - name: Checkout base commit
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.base_ref }}
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Get base docstrings
+        id: base-docstrings
+        run: |
+          CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
+          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout HEAD commit
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Get HEAD docstrings
+        id: head-docstrings
+        run: |
+          CHECKSUM=$(python .github/utils/docstrings_checksum.py --root "${{ github.workspace }}")
+          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"
+
+      - name: Write result
+        # The follow-up workflow reads these values; it never checks out the PR code.
+        run: |
+          mkdir -p result
+          echo "${{ github.event.pull_request.number }}" > result/pr_number
+          if [ "${{ steps.base-docstrings.outputs.checksum }}" != "${{ steps.head-docstrings.outputs.checksum }}" ]; then
+            echo "true" > result/should_label
+          else
+            echo "false" > result/should_label
+          fi
+
+      - name: Upload result
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: docstring-label-result
+          path: result/
+          retention-days: 1

.github/workflows/docstring_labeler_apply.yml

@@ -0,0 +1,42 @@
+name: Apply docstrings label
+
+# Runs after "Detect docstrings edit" completes. This workflow holds the write
+# token but only consumes the data artifact produced by the untrusted run — it
+# never checks out PR code, so it is safe under the workflow_run trigger.
+on:
+  workflow_run:
+    workflows: ["Detect docstrings edit"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  label:
+    runs-on: ubuntu-slim
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download result
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: docstring-label-result
+          path: result
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add label
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          SHOULD_LABEL=$(cat result/should_label)
+          PR_NUMBER=$(cat result/pr_number)
+          if [ "$SHOULD_LABEL" = "true" ]; then
+            gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label "type:documentation"
+          else
+            echo "Docstrings unchanged, nothing to label."
+          fi

.github/workflows/docusaurus_sync.yml

@@ -0,0 +1,66 @@
+name: Sync docs with Docusaurus
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pydoc/**"
+      - "haystack/**"
+      - ".github/workflows/docusaurus_sync.yml"
+
+env:
+  HATCH_VERSION: "1.16.5"
+  PYTHON_VERSION: "3.11"
+
+permissions:
+  contents: read
+
+jobs:
+  sync:
+    runs-on: ubuntu-slim
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Generate API reference for Docusaurus
+        run: hatch run docs
+
+      - name: Sync generated API reference to docs folder
+        run: |
+          SOURCE_PATH="tmp_api_reference"
+          DEST_PATH="docs-website/reference/haystack-api"
+
+          echo "Syncing from $SOURCE_PATH to $DEST_PATH"
+          mkdir -p $DEST_PATH
+          # Using rsync to copy files. This will also remove files in dest that are no longer in source.
+          rsync -av --delete --exclude='.git/' "$SOURCE_PATH/" "$DEST_PATH/"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Sync Haystack API reference on Docusaurus"
+          branch: sync-docusaurus-api-reference
+          base: main
+          title: "docs: sync Haystack API reference on Docusaurus"
+          add-paths: |
+            docs-website/reference/haystack-api
+          body: |
+            This PR syncs the Haystack API reference on Docusaurus. Just approve and merge it.

.github/workflows/e2e.yml

@@ -0,0 +1,52 @@
+name: end-to-end
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    paths:
+      - "e2e/**/*.py"
+      - ".github/workflows/e2e.yml"
+
+env:
+  PYTHON_VERSION: "3.10"
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  HATCH_VERSION: "1.16.5"
+  # we use HF_TOKEN instead of HF_API_TOKEN to work around a Hugging Face bug
+  # see https://github.com/deepset-ai/haystack/issues/9552
+  HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Run tests
+        run: hatch run test:e2e
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/github_release.yml

@@ -0,0 +1,104 @@
+name: Project release on Github
+
+on:
+  workflow_dispatch:  # this is useful to re-generate the release page without a new tag being pushed
+  push:
+    tags:
+      - "v2.[0-9]+.[0-9]+*"
+      # Ignore release versions tagged with -rc0 suffix
+      - "!v2.[0-9]+.[0-9]-rc0"
+
+permissions:
+  contents: read
+
+jobs:
+  generate-notes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # ncipollo/release-action creates the GitHub release
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-tags: true
+          fetch-depth: 0  # slow but needed by reno
+
+      - name: Parse version
+        id: version
+        run: |
+          echo "current_release=$(awk -F \\- '{print $1}' < VERSION.txt)" >> "$GITHUB_OUTPUT"
+          echo "current_pre_release=$(awk -F \\- '{print $2}' < VERSION.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Install reno
+        run: |
+          python -m pip install --upgrade pip
+          pip install "reno<5" --uploaded-prior-to=P1D
+
+      # Remove next version rc0 tag in the CI environment to prevent reno from assigning notes to future releases.
+      # This ensures release notes are correctly aggregated for the current version.
+      # This is a workaround. Can be removed if the release process is fully aligned with reno.
+      - name: Delete next version rc0 tag in the CI environment
+        run: |
+          # Parse version X.Y.Z and increment Y for next minor version
+          IFS='.' read -r MAJOR MINOR _ <<< "${{ steps.version.outputs.current_release }}"
+          NEXT_MINOR=$((MINOR + 1))
+          NEXT_TAG="v${MAJOR}.${NEXT_MINOR}.0-rc0"
+
+          if git rev-parse --verify "$NEXT_TAG" >/dev/null 2>&1; then
+            git tag -d "$NEXT_TAG"
+            echo "Deleted local tag $NEXT_TAG"
+          else
+            echo "Tag $NEXT_TAG does not exist locally"
+          fi
+
+      - name: Generate release notes
+        env:
+
+          EARLIEST_VERSION: v${{ steps.version.outputs.current_release }}-rc1
+        run: |
+          reno report --no-show-source --ignore-cache --earliest-version "$EARLIEST_VERSION" -o relnotes.rst
+
+      - name: Convert to Markdown
+        uses: docker://pandoc/core:3.8@sha256:c7127705b9d84c1b4acbbe411a86e3a6d67ac57870654dc378b6fe636ab747df
+        with:
+          args: "--from rst --to gfm --syntax-highlighting=none --wrap=none relnotes.rst -o relnotes.md"
+
+      # We copy the relnotes file since the original one cannot be modified due to permissions
+      - name: Copy relnotes file
+        run: |
+          cat relnotes.md > enhanced_relnotes.md
+
+      - name: Add contributor list
+          # only for minor releases and minor release candidates (not bugfix releases)
+        if: endsWith(steps.version.outputs.current_release, '.0')
+        env:
+          GH_TOKEN: ${{ github.token }}
+          START: v${{ steps.version.outputs.current_release }}-rc0
+          END: ${{ github.ref_name }}
+        run: |
+          JQ_EXPR='[.commits[].author.login]
+            | map(select(. != null and . != "HaystackBot" and . != "dependabot[bot]" and . != "github-actions[bot]"))
+            | unique
+            | sort_by(ascii_downcase)
+            | map("@\(.)")
+            | join(", ")'
+          CONTRIBUTORS=$(gh api "repos/deepset-ai/haystack/compare/$START...$END" \
+            --jq "$JQ_EXPR") || { echo "Unable to fetch contributors"; exit 1; }
+
+          {
+            echo ""
+            echo "## 💙 Big thank you to everyone who contributed to this release!"
+            echo ""
+            echo "$CONTRIBUTORS"
+          } >> enhanced_relnotes.md
+
+      - name: Debug
+        run: |
+          cat enhanced_relnotes.md
+
+      - uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0
+        with:
+          bodyFile: "enhanced_relnotes.md"
+          prerelease: ${{ steps.version.outputs.current_pre_release != '' }}
+          allowUpdates: true

.github/workflows/labeler.yml

@@ -0,0 +1,17 @@
+name: "Labeler"
+on:
+- pull_request_target
+
+permissions:
+  contents: read
+
+jobs:
+  triage:
+    runs-on: ubuntu-slim
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+    - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/license_compliance.yml

@@ -0,0 +1,69 @@
+name: License Compliance
+
+on:
+  pull_request:
+    paths:
+      - "**/pyproject.toml"
+      - ".github/workflows/license_compliance.yml"
+  # Since we test PRs, there is no need to run the workflow at each
+  # merge on `main`. Let's use a cron job instead.
+  schedule:
+    - cron: "0 0 * * *" # every day at midnight
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  license_check_direct:
+    name: Direct dependencies only
+    env:
+      REQUIREMENTS_FILE: requirements_direct.txt
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Get direct dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install toml --uploaded-prior-to=P1D
+          python .github/utils/pyproject_to_requirements.py pyproject.toml > ${{ env.REQUIREMENTS_FILE }}
+
+      - name: Check Licenses
+        id: license_check_report
+        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          requirements: ${{ env.REQUIREMENTS_FILE }}
+          fail: "Copyleft,Other,Error"
+          # Exclusions in the vanilla distribution must be explicitly motivated
+          # - tqdm is MLP but there are no better alternatives
+          # - typing_extensions>=4.13.0 has a Python Software Foundation License 2.0 but pip-license-checker does not recognize it
+          #   (https://github.com/pilosus/pip-license-checker/issues/143)
+          exclude: "(?i)^(tqdm|typing_extensions).*"
+
+      # We keep the license inventory on FOSSA
+      - name: Send license report to Fossa
+        uses: fossas/fossa-action@29693cc50323968e039056be419b32989fc5880c # v2.0.0
+        continue-on-error: true # not critical
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+      - name: Print report
+        if: ${{ always() }}
+        run: echo "${{ steps.license_check_report.outputs.report }}"
+
+      - name: Notify Slack on failure
+        if: failure()
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

.github/workflows/nightly_testpypi_release.yml

@@ -0,0 +1,51 @@
+name: Nightly pre-release on PyPI
+
+on:
+  schedule:
+    # Run at midnight UTC every day
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  nightly-release:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    # Always build from main for consistency (scheduled and manual runs)
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          persist-credentials: false
+          ref: main
+          fetch-depth: 1
+
+      # Reads VERSION.txt, strips any -rcN suffix, and appends .devYYYYMMDDHHMMSS
+      # (e.g. 2.25.0.dev20250217000000) so each run gets a unique, PEP 440–valid pre-release version.
+      - name: Set nightly version
+        id: set-version
+        run: |
+          BASE_VERSION=$(sed 's/-rc[0-9]*$//' VERSION.txt)
+          TIMESTAMP=$(date +%Y%m%d%H%M%S)
+          NIGHTLY_VERSION="${BASE_VERSION}.dev${TIMESTAMP}"
+          echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "${NIGHTLY_VERSION}" > VERSION.txt
+          echo "Building haystack-ai version: ${NIGHTLY_VERSION}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Build Haystack
+        run: hatch build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

.github/workflows/project.yml

@@ -0,0 +1,19 @@
+name: Track issues with Github project
+
+on:
+  issues:
+    types:
+      - opened
+
+permissions:
+  contents: read
+
+jobs:
+  add-to-project:
+    name: Add new issues to project for triage
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
+        with:
+          project-url: https://github.com/orgs/deepset-ai/projects/5
+          github-token: ${{ secrets.GH_PROJECT_PAT }}

.github/workflows/promote_unstable_docs.yml

@@ -0,0 +1,65 @@
+name: Release new minor version docs
+
+on:
+  push:
+    tags:
+      # Trigger this only for new minor version tags (e.g. v2.99.0)
+      - "v[0-9]+.[0-9]+.0"
+      # Exclude 1.x tags
+      - "!v1.[0-9]+.[0-9]+"
+
+env:
+  PYTHON_VERSION: "3.10"
+
+permissions:
+  contents: read
+
+jobs:
+  promote:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout this repo
+        # use VERSION.txt file from main branch
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          ref: main
+
+      - name: Get version to release
+        id: version
+        shell: bash
+        # We only need `major.minor`. At this point, VERSION.txt contains the next version.
+        # For example, if we are releasing 2.20.0, VERSION.txt contains 2.21.0-rc0.
+        run: |
+          MAJOR=$(cut -d "." -f 1 < VERSION.txt)
+          MINOR=$(cut -d "." -f 2 < VERSION.txt)
+          MINOR=$((MINOR - 1))
+          echo "version=${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Promote unstable docs for Docusaurus
+        run: |
+          python ./.github/utils/promote_unstable_docs_docusaurus.py --version ${{ steps.version.outputs.version }}
+
+      - name: Create Pull Request with Docusaurus docs updates
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          branch: promote-unstable-docs-${{ steps.version.outputs.version }}
+          base: main
+          title: "docs: promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
+          add-paths: |
+            docs-website
+          body: |
+            This PR promotes the unstable docs for Haystack ${{ steps.version.outputs.version }} to stable.
+            It is expected to run at the time of the release.
+            You can inspect the docs preview and merge it. There should now be only one unstable version representing the next (main) branch.
+          # This workflow is triggered by a tag pushed by the HaystackBot in release.yml > create-release-tag.
+          # GitHub requires reviewers to be different from the PR author, so setting `github.actor`
+          # would fail (it would request a review from the HaystackBot itself).
+          # So we don't set any reviewers and instead notify the Release Manager
+          # (see .github/utils/prepare_release_notification.sh).

.github/workflows/push_release_notes_to_website.yml

@@ -0,0 +1,65 @@
+name: Push release notes to website
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Haystack version (vX.Y.Z)'
+        required: true
+        type: string
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  VERSION: ${{ inputs.version }}
+
+permissions:
+  contents: read
+
+jobs:
+  push-release-notes-to-website:
+
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout Haystack home repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/haystack-home
+
+      - name: Get release notes and add frontmatter
+        id: release_notes
+
+        run: |
+          VERSION_NUMBER="${VERSION:1}"
+          RELEASE_DATE=$(gh release view "$VERSION" --repo deepset-ai/haystack --json publishedAt --jq '.publishedAt | split("T")[0]')
+          RELEASE_NOTES_PATH="content/release-notes/$VERSION_NUMBER.md"
+
+          {
+            echo "---"
+            echo "title: Haystack $VERSION_NUMBER"
+            echo "description: Release notes for Haystack $VERSION_NUMBER"
+            echo "toc: True"
+            echo "date: $RELEASE_DATE"
+            echo "last_updated: $RELEASE_DATE"
+            echo 'tags: ["Release Notes"]'
+            echo "link: https://github.com/deepset-ai/haystack/releases/tag/$VERSION"
+            echo "---"
+            echo ""
+          } > "$RELEASE_NOTES_PATH"
+
+          gh release view "$VERSION" --repo deepset-ai/haystack --json body --jq '.body' >> "$RELEASE_NOTES_PATH"
+
+          cat "$RELEASE_NOTES_PATH"
+
+      - name: Create Pull Request to Haystack Home
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Add release notes for Haystack ${{ env.VERSION }}"
+          branch: add-release-notes-for-haystack-${{ env.VERSION }}
+          base: main
+          title: "docs: add release notes for Haystack ${{ env.VERSION }}"
+          add-paths: |
+            content/release-notes
+          body: |
+            This PR adds the release notes for Haystack ${{ env.VERSION }} to the website.
+          reviewers: "${{ github.actor }}"

.github/workflows/pypi_release.yml

@@ -0,0 +1,38 @@
+name: Project release on PyPi
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+      # We must not release versions tagged with -rc0 suffix
+      - "!v[0-9]+.[0-9]+.[0-9]-rc0"
+
+env:
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  release-on-pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Build Haystack
+        run: hatch build
+
+      - name: Publish on PyPi
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

.github/workflows/release.yml

@@ -0,0 +1,302 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., v2.99.0-rc1 or v2.99.0)'
+        required: true
+        type: string
+
+# Only one release workflow runs at a time; additional runs are queued.
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  authorize:
+    # Releasing acts as HaystackBot (a ruleset-bypass actor), so the ruleset
+    # cannot enforce *who* started the release. Gate on the triggering user's
+    # role instead: only maintainers/admins may run this workflow, even though
+    # workflow_dispatch itself is available to anyone with write access.
+    runs-on: ubuntu-slim
+    steps:
+      - name: Verify the triggering user may release
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          ACTOR: ${{ github.triggering_actor }}
+        run: |
+          ROLE=$(gh api "repos/${{ github.repository }}/collaborators/${ACTOR}/permission" --jq '.role_name')
+          echo "::notice::${ACTOR} has repository role '${ROLE}'"
+          case "$ROLE" in
+            admin|maintain)
+              echo "✅ ${ACTOR} is authorized to trigger a release." ;;
+            *)
+              echo "::error::${ACTOR} has role '${ROLE}'. A release can only be triggered by a user with the 'maintain' or 'admin' role."
+              exit 1 ;;
+          esac
+
+  parse-validate-version:
+    needs: ["authorize"]
+    runs-on: ubuntu-slim
+    outputs:
+      version: ${{ steps.parse-validate.outputs.version }}
+      major_minor: ${{ steps.parse-validate.outputs.major_minor }}
+      release_branch: ${{ steps.parse-validate.outputs.release_branch }}
+      is_rc: ${{ steps.parse-validate.outputs.is_rc }}
+      is_first_rc: ${{ steps.parse-validate.outputs.is_first_rc }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+      - name: Parse and validate version
+        id: parse-validate
+        env:
+          GH_TOKEN: ${{ github.token }}
+          VERSION: ${{ inputs.version }}
+        run: .github/utils/parse_validate_version.sh "$VERSION"
+
+  branch-off:
+    needs: ["parse-validate-version"]
+    if: needs.parse-validate-version.outputs.is_first_rc == 'true'
+    uses: ./.github/workflows/branch_off.yml
+    # https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows#passing-secrets-to-nested-workflows
+    secrets: inherit
+
+  create-release-tag:
+    needs: ["parse-validate-version", "branch-off"]
+    if: always() && needs.parse-validate-version.result == 'success' && needs.branch-off.result != 'failure'
+    runs-on: ubuntu-slim
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+          # use this token so the created tag triggers workflows (does not happen with the default github.token)
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+      - name: Update VERSION.txt and create tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          git checkout ${{ needs.parse-validate-version.outputs.release_branch }}
+          git pull origin ${{ needs.parse-validate-version.outputs.release_branch }}
+
+          echo "${{ needs.parse-validate-version.outputs.version }}" > VERSION.txt
+          git add VERSION.txt
+          git commit -m "bump version to ${{ needs.parse-validate-version.outputs.version }}"
+          git push origin ${{ needs.parse-validate-version.outputs.release_branch }}
+
+          TAG="v${{ needs.parse-validate-version.outputs.version }}"
+          git tag -m "$TAG" "$TAG"
+          git push origin "$TAG"
+
+  check-artifacts:
+    needs: ["parse-validate-version", "create-release-tag"]
+    if: always() && needs.parse-validate-version.result == 'success' && needs.create-release-tag.result == 'success'
+    runs-on: ubuntu-latest
+    outputs:
+      github_url: ${{ steps.set-outputs.outputs.github_url }}
+      pypi_url: ${{ steps.set-outputs.outputs.pypi_url }}
+      docker_url: ${{ steps.set-outputs.outputs.docker_url }}
+    env:
+      GH_TOKEN: ${{ github.token }}
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0  # needed to fetch tags and branches
+
+      - name: Wait for release workflows
+        run: |
+          .github/utils/wait_for_workflows.sh "v${{ env.VERSION }}" \
+            "Project release on PyPi" \
+            "Project release on Github" \
+            "Docker image release"
+
+      - name: Check artifacts
+        run: |
+          check() {
+            for _ in {1..5}; do curl -sf "$2" > /dev/null && echo "✅ $1" && return 0; sleep 30; done
+            echo "❌ $1 not found" && return 1
+          }
+          check "GitHub Release" "https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.VERSION }}"
+          check "PyPI package" "https://pypi.org/pypi/haystack-ai/${{ env.VERSION }}/json"
+          check "Docker image" "https://hub.docker.com/v2/repositories/deepset/haystack/tags/base-v${{ env.VERSION }}"
+
+      - name: Set artifact URLs
+        id: set-outputs
+        run: |
+          {
+            echo "github_url=https://github.com/${{ github.repository }}/releases/tag/v${{ env.VERSION }}"
+            echo "pypi_url=https://pypi.org/project/haystack-ai/${{ env.VERSION }}/"
+            echo "docker_url=https://hub.docker.com/r/deepset/haystack/tags?name=base-v${{ env.VERSION }}"
+          } >> "$GITHUB_OUTPUT"
+
+  bump-dc-pipeline-templates:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout dc-pipeline-templates
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/dc-pipeline-templates
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Update haystack pin
+        run: sed -i "s/haystack-ai>=.*/haystack-ai>=${VERSION}/" requirements-test.txt
+
+      - name: Create Pull Request
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Bump haystack to ${{ env.VERSION }}"
+          branch: bump-haystack-${{ env.VERSION }}
+          base: main
+          title: "chore: bump haystack to ${{ env.VERSION }}"
+          add-paths: requirements-test.txt
+          body: |
+            Bump haystack pin to `${{ env.VERSION }}` for platform testing.
+
+  bump-deepset-cloud-custom-nodes:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Checkout haystack (for utils)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          path: haystack
+          sparse-checkout: .github/utils/update_haystack_dc_custom_nodes.py
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout deepset-cloud-custom-nodes
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/deepset-cloud-custom-nodes
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          path: deepset-cloud-custom-nodes
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.13"
+
+      - name: Install tomlkit
+        run: |
+          python -m pip install --upgrade pip
+          pip install tomlkit --uploaded-prior-to=P1D
+
+      - name: Update haystack-ai in uv.lock
+        run: python haystack/.github/utils/update_haystack_dc_custom_nodes.py "${{ env.VERSION }}" deepset-cloud-custom-nodes/uv.lock
+
+      - name: Create Pull Request
+        id: create-pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          path: deepset-cloud-custom-nodes
+          commit-message: "Bump haystack to ${{ env.VERSION }}"
+          branch: bump-haystack-${{ env.VERSION }}
+          base: main
+          title: "chore: bump haystack to ${{ env.VERSION }}"
+          add-paths: uv.lock
+          body: |
+            Bump haystack pin to `${{ env.VERSION }}` for platform testing.
+
+  bump-haystack-runtime:
+    needs: ["parse-validate-version", "check-artifacts"]
+    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
+    runs-on: ubuntu-slim
+    outputs:
+      pr_url: ${{ steps.wait-pr.outputs.pr_url }}
+    env:
+      VERSION: ${{ needs.parse-validate-version.outputs.version }}
+    steps:
+      - name: Trigger "Update package version" workflow
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          gh workflow run update-package-version.yaml \
+            -R deepset-ai/haystack-runtime \
+            -f haystack_version="${{ env.VERSION }}"
+
+      - name: Wait for PR
+        id: wait-pr
+        env:
+          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+        run: |
+          TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          for i in $(seq 1 30); do
+            PR_URL=$(gh pr list -R deepset-ai/haystack-runtime \
+              --head "bump/hs${{ env.VERSION }}" \
+              --json url,createdAt \
+              --jq ".[] | select(.createdAt >= \"$TRIGGER_TIME\") | .url")
+            if [[ -n "$PR_URL" ]]; then
+              echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+              echo "Found PR: $PR_URL"
+              exit 0
+            fi
+            echo "Attempt $i: PR not found yet, waiting 10s..."
+            sleep 10
+          done
+          echo "PR not found after 5 minutes"
+
+  notify:
+    needs:
+      - "parse-validate-version"
+      - "branch-off"
+      - "create-release-tag"
+      - "check-artifacts"
+      - "bump-dc-pipeline-templates"
+      - "bump-deepset-cloud-custom-nodes"
+      - "bump-haystack-runtime"
+    if: always()
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Prepare release notification
+        env:
+          VERSION: ${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          HAS_FAILURE: ${{ contains(needs.*.result, 'failure') }}
+          IS_RC: ${{ needs.parse-validate-version.outputs.is_rc }}
+          IS_FIRST_RC: ${{ needs.parse-validate-version.outputs.is_first_rc }}
+          MAJOR_MINOR: ${{ needs.parse-validate-version.outputs.major_minor }}
+          GITHUB_URL: ${{ needs.check-artifacts.outputs.github_url }}
+          PYPI_URL: ${{ needs.check-artifacts.outputs.pypi_url }}
+          DOCKER_URL: ${{ needs.check-artifacts.outputs.docker_url }}
+          BUMP_VERSION_PR_URL: ${{ needs.branch-off.outputs.bump_version_pr_url }}
+          DC_PIPELINE_TEMPLATES_PR_URL: ${{ needs.bump-dc-pipeline-templates.outputs.pr_url }}
+          DC_CUSTOM_NODES_PR_URL: ${{ needs.bump-deepset-cloud-custom-nodes.outputs.pr_url }}
+          HAYSTACK_RUNTIME_PR_URL: ${{ needs.bump-haystack-runtime.outputs.pr_url }}
+        run: .github/utils/prepare_release_notification.sh
+
+      - name: Send release notification to Slack
+        uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }}
+          webhook-type: incoming-webhook
+          payload-file-path: slack_payload.json

.github/workflows/release_notes.yml

@@ -0,0 +1,85 @@
+name: Check Release Notes
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+      - labeled
+      - unlabeled
+    paths:
+      - "**.py"
+      - "pyproject.toml"
+      - "!.github/**/*.py"
+      - "releasenotes/notes/*.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  reno:
+    runs-on: ubuntu-slim
+    env:
+      PYTHON_VERSION: "3.10"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+      - name: Get release note files
+        id: changed-files
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
+        with:
+          files: releasenotes/notes/*.yaml
+
+      - name: Check release notes
+        if: steps.changed-files.outputs.any_changed == 'false' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        run: |
+          # Check if any of the commit messages contain tags ci/docs/test
+          if git log --pretty=%s origin/main..HEAD | grep -E '^(ci:|docs:|test:)' > /dev/null; then
+            echo "Skipping release note check for commits with 'ci:', 'docs:', or 'test:' tags."
+          else
+            echo "::error::The release notes file is missing, please add one or attach the label 'ignore-for-release-notes' to this PR."
+            exit 1
+          fi
+
+      - name: Verify release notes formatting
+        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        run: |
+          python -m pip install --upgrade pip
+          pip install "reno<5" --uploaded-prior-to=P1D
+          reno lint .  # it is not possible to pass a list of files to reno lint
+
+      - name: Check reStructuredText code formatting
+        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
+        shell: python
+        run: |
+          files = "${{ steps.changed-files.outputs.all_changed_files }}".split()
+          errors = []
+
+          for filepath in files:
+            with open(filepath) as f:
+              for line_no, line in enumerate(f, start=1):
+                # Check for triple backticks (Markdown code blocks)
+                if "```" in line:
+                  err = (f"Format error in {filepath}:{line_no}: "
+                         "Found triple backticks. Use reStructuredText code block directive instead: .. code:: python")
+                  errors.append(err)
+
+                # Check for single backticks (Markdown inline code)
+                if "`" in line.replace("```", "").replace("``", ""):
+                  err = (f"Format error in {filepath}:{line_no}: "
+                         "Found single backticks. Use double backticks (``code``) for inline code.")
+                  errors.append(err)
+
+          if errors:
+              raise Exception("\n".join(errors))

.github/workflows/release_notes_skipper.yml

@@ -0,0 +1,26 @@
+name: Check Release Notes
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+      - labeled
+      - unlabeled
+    paths-ignore:
+      - "**.py"
+      - "pyproject.toml"
+      - "!.github/**/*.py"
+      - "releasenotes/notes/*.yaml"
+
+permissions:
+  contents: read
+
+jobs:
+  reno:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Skip mandatory job
+        run: echo "Skipped!"

.github/workflows/scorecard.yml

@@ -0,0 +1,66 @@
+# OpenSSF Scorecard
+#
+# Runs the OpenSSF Scorecard analyzer (https://github.com/ossf/scorecard) and
+# publishes results to GitHub's Security tab via the code-scanning SARIF API.
+#
+# Why: Scorecard gives haystack maintainers a single weekly signal on
+# supply-chain posture (branch protection, SAST coverage, dependency-update
+# tool, signed releases, etc.) without changing any existing CI behaviour.
+# It also lets downstream consumers verify the project's score on the
+# OpenSSF deps.dev / Security Insights dashboards.
+#
+# Permissions are scoped to the minimum required by the action:
+#   - security-events: write   -> upload SARIF
+#   - id-token:        write   -> publish to the public OpenSSF API (badge)
+#   - contents:        read    -> default repo read
+#   - actions:         read    -> read CI config for checks
+#
+# Default branch is 'main'.
+
+name: OpenSSF Scorecard
+
+on:
+  # Run weekly so results stay fresh, plus on branch-protection changes so the
+  # branch-protection check is re-evaluated immediately in the Security tab.
+  branch_protection_rule:
+  schedule:
+    - cron: '32 6 * * 1'
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+      contents: read
+      actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Run analysis
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # Publish results to the OpenSSF public REST API so the badge in
+          # the README and deps.dev card stay up to date. Opt-in.
+          publish_results: true
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      - name: Upload to code-scanning
+        uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+        with:
+          sarif_file: results.sarif

.github/workflows/slow.yml

@@ -0,0 +1,130 @@
+name: Slow Integration Tests
+
+# The workflow will always run, but the actual tests will only execute when:
+# - The workflow is triggered manually
+# - The workflow is scheduled
+# - The PR has the "run-slow-tests" label
+# - The push is to a release branch
+# - There are changes to relevant files.
+# Note: If no conditions are met, the workflow will complete successfully without running tests
+# to satisfy Branch Protection rules.
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+  PYTHON_VERSION: "3.10"
+  HATCH_VERSION: "1.16.5"
+  HAYSTACK_MPS_ENABLED: false
+  HAYSTACK_XPU_ENABLED: false
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  schedule:
+    - cron: "0 0 * * *"
+  push:
+    branches:
+      # release branches have the form v1.9.x
+      - "v[0-9].*[0-9].x"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+      - unlabeled
+
+permissions:
+  contents: read
+
+jobs:
+  check-if-changed:
+    # This job checks if the relevant files have been changed.
+    # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
+    # This ensures the "Slow Integration Tests completed" job always runs, which is required by Branch Protection rules.
+    name: Check if changed
+    runs-on: ubuntu-slim
+    permissions:
+      pull-requests: read
+    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
+    outputs:
+      changes: ${{ steps.changes.outputs.changes }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Check for changed code
+        id: changes
+        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        with:
+          # List of Python files that trigger slow integration tests when modified
+          filters: |
+            changes:
+              - "haystack/components/evaluators/sas_evaluator.py"
+              - "haystack/components/generators/openai_dalle.py"
+              - "haystack/components/preprocessors/embedding_based_document_splitter.py"
+              - "haystack/components/retrievers/multi_query_embedding_retriever.py"
+
+              - "test/components/evaluators/test_sas_evaluator.py"
+              - "test/components/generators/test_openai_dalle.py"
+              - "test/components/preprocessors/test_embedding_based_document_splitter.py"
+              - "test/components/retrievers/test_multi_query_embedding_retriever.py"
+
+  slow-integration-tests:
+    name: Slow Tests / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: check-if-changed
+    timeout-minutes: 30
+    # Run tests if: manual trigger, scheduled, PR has label, release branch, or relevant files changed
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'schedule' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests')) ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/v[0-9].*[0-9].x') ||
+      (needs.check-if-changed.outputs.changes == 'true')
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Run tests
+        run: hatch run test:integration-only-slow
+
+      - name: Notify Slack on nightly failure
+        if: failure() && github.event_name == 'schedule'
+        uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}
+
+  slow-integration-tests-completed:
+    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
+    name: Slow Integration Tests completed
+    runs-on: ubuntu-slim
+    if: ${{ always() && !cancelled() }}
+    needs: slow-integration-tests
+
+    steps:
+      - name: Mark tests as completed
+        run: |
+          if [ "${{ needs.slow-integration-tests.result }}" = "failure" ]; then
+            echo "Slow Integration Tests failed!"
+            exit 1
+          else
+            echo "Slow Integration Tests completed!"
+          fi

.github/workflows/stale.yml

@@ -0,0 +1,21 @@
+name: 'Stalebot'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  makestale:
+    runs-on: ubuntu-slim
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
+        with:
+          any-of-labels: 'proposal,information-needed'
+          stale-pr-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
+          days-before-stale: 30
+          days-before-close: 10

.github/workflows/tests.yml

@@ -0,0 +1,343 @@
+name: Tests
+
+# The workflow will always run, but the actual tests will only execute when:
+# - The workflow is triggered manually.
+# - The push is to main or a release branch.
+# - There are changes to relevant files on a pull request.
+# Note: If no conditions are met, the workflow will complete successfully without running tests
+# to satisfy Branch Protection rules.
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  push:
+    branches:
+      - main
+      # release branches have the form v1.9.x
+      - "v[0-9].*[0-9].x"
+    # when we push, we do not need to satisfy Branch Protection rules, so we can ignore PRs that just change docs
+    paths-ignore:
+      - "docs/**"
+      - "docs-website/**"
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
+  CORE_AZURE_CS_API_KEY: ${{ secrets.CORE_AZURE_CS_API_KEY }}
+  AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+  AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
+  PYTHON_VERSION: "3.10"
+  HATCH_VERSION: "1.16.5"
+
+permissions:
+  contents: read
+
+jobs:
+  check-if-changed:
+    # This job checks if the relevant files have been changed.
+    # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
+    # This ensures the "Mark tests as completed" job always runs, which is required by Branch Protection rules.
+    name: Check if changed
+    runs-on: ubuntu-slim
+    permissions:
+      pull-requests: read
+    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
+    outputs:
+      changes: ${{ steps.changes.outputs.changes }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Check for changed code
+        id: changes
+        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        with:
+          filters: |
+            changes:
+              - "haystack/**/*.py"
+              - "test/**/*.py"
+              - "pyproject.toml"
+              - ".github/utils/*.py"
+              - "scripts/*.py"
+
+  format:
+    needs: check-if-changed
+    # Run tests if: manual trigger, push to main/release, or relevant files changed
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'push' ||
+      (needs.check-if-changed.outputs.changes == 'true')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Ruff - check format and linting
+        run: hatch run fmt-check
+
+      - name: Check presence of license header
+        run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye:v6.5.1@sha256:0ebd72353053aa6d3f0cb44a6897786386347bd4636ffc9592f8e47c81ea2b9d check
+
+  check-imports:
+    needs: format
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+
+      - name: Check imports
+        run: hatch run python .github/utils/check_imports.py
+
+  unit-tests:
+    name: Unit / ${{ matrix.os }}
+    needs: format
+    timeout-minutes: 30
+    permissions:
+      contents: write
+      pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: true # needed for python-coverage-comment-action
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:unit
+
+      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: cache
+        if: matrix.os == 'macos-latest'
+        with:
+          path: ${{ steps.hatch.outputs.env }}
+          key: ${{ runner.os }}-${{ github.sha }}
+
+      # On PR: posts coverage comment (directly on same-repo PRs; via artifact for fork PRs).
+      # On push to main: stores coverage baseline on the data branch.
+      # We only upload coverage for ubuntu-latest, as handling multiple OSes adds complexity for little gain.
+      - name: Coverage comment
+        id: coverage_comment
+        if: matrix.os == 'ubuntu-latest'
+        uses: py-cov-action/python-coverage-comment-action@5d8df5979747514c914e1c5a12335a7cf9a2745f # v4.1
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          MINIMUM_GREEN: 90
+          MINIMUM_ORANGE: 60
+
+      - name: Upload coverage comment to be posted
+        if: matrix.os == 'ubuntu-latest' && github.event_name == 'pull_request' && steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt
+
+  mypy:
+    needs: unit-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+          # With the default value of 1, there are corner cases where tj-actions/changed-files
+          # fails with a `no merge base` error
+          fetch-depth: 0
+      - name: Get changed files
+        id: files
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
+        with:
+          files: |
+            **/*.py
+            pyproject.toml
+          files_ignore: |
+            test/**
+            .github/**
+            scripts/**
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        if: steps.files.outputs.any_changed == 'true'
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Mypy
+        if: steps.files.outputs.any_changed == 'true'
+        run: |
+          mkdir .mypy_cache
+          hatch run test:types
+
+  integration-tests-linux:
+    name: Integration / ubuntu-latest
+    needs: unit-tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  integration-tests-macos:
+    name: Integration / macos-latest
+    needs: unit-tests
+    runs-on: macos-latest
+    timeout-minutes: 30
+    env:
+      HAYSTACK_MPS_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        id: cache
+        with:
+          path: ${{ steps.hatch.outputs.env }}
+          key: ${{ runner.os }}-${{ github.sha }}
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  integration-tests-windows:
+    name: Integration / windows-latest
+    needs: unit-tests
+    runs-on: windows-latest
+    timeout-minutes: 30
+    env:
+      HAYSTACK_XPU_ENABLED: false
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install Hatch
+        id: hatch
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==${{ env.HATCH_VERSION }} --uploaded-prior-to=P1D
+          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"
+
+      - name: Run
+        run: hatch run test:integration-only-fast
+
+  notify-slack-on-failure:
+    if: failure() && github.ref_name == 'main'
+    needs:
+      - check-imports
+      - mypy
+      - integration-tests-linux
+      - integration-tests-macos
+      - integration-tests-windows
+    runs-on: ubuntu-slim
+    steps:
+      - uses: deepset-ai/notify-slack-action@a65def0c8bf91d6520286ab34280151c76a5a008 # v1.1.0
+        with:
+          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}
+
+  tests-completed:
+    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
+    name: Mark tests as completed
+    runs-on: ubuntu-slim
+    if: ${{ always() && !cancelled() }}
+    needs:
+      - check-imports
+      - mypy
+      - integration-tests-linux
+      - integration-tests-macos
+      - integration-tests-windows
+
+    steps:
+      - name: Mark tests as completed
+        run: |
+          if [ "${{ needs.check-imports.result }}" = "failure" ] ||
+             [ "${{ needs.mypy.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-linux.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-macos.result }}" = "failure" ] ||
+             [ "${{ needs.integration-tests-windows.result }}" = "failure" ]; then
+            echo "Tests failed!"
+            exit 1
+          else
+            echo "Tests completed!"
+          fi

.github/workflows/update-platform-components.yml

@@ -0,0 +1,99 @@
+name: Update Platform Components Table
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: update-platform-components
+  cancel-in-progress: false
+
+env:
+  PYTHON_VERSION: "3.11"
+  OUTPUT_FILE: docs-website/docs/overview/platform-components.mdx
+
+permissions:
+  contents: read
+
+jobs:
+  update:
+    runs-on: ubuntu-slim
+
+    steps:
+      - name: Checkout deepset-ai/haystack
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          # Full history so git diff against HEAD works correctly
+          fetch-depth: 0
+
+      - name: Checkout deepset-ai/haystack-core-integrations
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: deepset-ai/haystack-core-integrations
+          path: haystack-core-integrations
+          fetch-depth: 1
+          sparse-checkout: integrations
+          sparse-checkout-cone-mode: true
+
+      - name: Checkout platform repo (sparse — schema file only)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: ${{ secrets.SCHEMA_REPO }}
+          token: ${{ secrets.PLATFORM_REPO_PAT }}
+          path: platform-repo
+          sparse-checkout: |
+            ${{ secrets.SCHEMA_REPO_PATH }}
+          sparse-checkout-cone-mode: false
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Generate platform components table
+        run: |
+          python scripts/generate_platform_components_table.py \
+            --haystack-src . \
+            --integrations-src ./haystack-core-integrations \
+            --schema ./platform-repo/${{ secrets.SCHEMA_REPO_PATH }} \
+            --output ./${{ env.OUTPUT_FILE }}
+
+      - name: Compute PR body
+        id: pr_body
+        run: |
+          FILE="${{ env.OUTPUT_FILE }}"
+          if git ls-files --error-unmatch "$FILE" 2>/dev/null; then
+            ADDED=$(git diff HEAD -- "$FILE" | grep -cE '^\+\| `' || true)
+            REMOVED=$(git diff HEAD -- "$FILE" | grep -cE '^\-\| `' || true)
+            SUMMARY="Component rows changed: **+${ADDED}** added, **-${REMOVED}** removed."
+          else
+            SUMMARY="Initial generation of the platform components table."
+          fi
+
+          {
+            echo "body<<EOF"
+            echo "## Platform Components Table Update"
+            echo ""
+            echo "${SUMMARY}"
+            echo ""
+            echo "_Auto-generated by the [update-platform-components](https://github.com/deepset-ai/haystack/actions/workflows/update-platform-components.yml) workflow._"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Open or update pull request
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "chore: update platform components table"
+          branch: chore/update-platform-components
+          base: main
+          title: "chore: update platform components table"
+          add-paths: |
+            ${{ env.OUTPUT_FILE }}
+          body: ${{ steps.pr_body.outputs.body }}
+          labels: |
+            documentation
+            ignore-for-release-notes

.github/workflows/workflows_linting.yml

@@ -0,0 +1,30 @@
+name: Github workflows linter
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/**"
+
+permissions:
+  contents: read
+
+jobs:
+  lint-workflows:
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version: ">=1.24.0"
+
+      - name: Install actionlint
+        run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12
+
+      - name: Run actionlint
+        env:
+          SHELLCHECK_OPTS: --exclude=SC2102
+        run: actionlint