name: Create Release

on:
  push:
    branches:
      - main  # Automatically create release when merging to main
    tags:
      - 'v*.*.*'  # Also support manual version tags
  workflow_dispatch:  # Manual trigger for re-runs (no inputs needed)

permissions: {}  # Minimal permissions at workflow level for OSSF Scorecard

# Prevent multiple full pipeline runs from competing for runners simultaneously.
# cancel-in-progress: false ensures the active run completes; new runs queue (at most
# 1 in-progress + 1 pending). Tag pushes get their own group (refs/tags/v1.2.3).
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  # ============================================================================
  # VERSION CHECK - Skip expensive jobs when the release already exists
  # ============================================================================
  # Lightweight pre-flight: reads __version__.py, checks if a GitHub release
  # for that tag already exists.  Tag pushes and workflow_dispatch bypass the
  # check (always proceed).  All gate jobs depend on should_release == 'true'.
  # ============================================================================
  version-check:
    runs-on: ubuntu-latest
    outputs:
      should_release: ${{ steps.check.outputs.should_release }}
    permissions:
      contents: read
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Sparse checkout version file
        if: ${{ !(github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/'))) }}
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          sparse-checkout: src/local_deep_research/__version__.py
          sparse-checkout-cone-mode: false
          fetch-depth: 1
          persist-credentials: false

      - name: Check if release is needed
        id: check
        env:
          EVENT_NAME: ${{ github.event_name }}
          GIT_REF: ${{ github.ref }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
        run: |
          # Tag pushes and manual triggers always proceed
          if [[ "$EVENT_NAME" = "workflow_dispatch" ]] || [[ "$EVENT_NAME" = "push" && "$GIT_REF" =~ ^refs/tags/ ]]; then
            echo "should_release=true" >> "$GITHUB_OUTPUT"
            echo "Bypass: $EVENT_NAME on $GIT_REF — always proceed"
            exit 0
          fi

          # For main branch pushes: check if version has an existing release
          VERSION=$(grep -oP '(?<=__version__ = ")[^"]*' src/local_deep_research/__version__.py)
          if [[ -z "$VERSION" ]]; then
            echo "ERROR: Could not extract version from __version__.py" >&2
            exit 1
          fi
          TAG="v${VERSION}"
          echo "Detected version: $VERSION (tag: $TAG)"

          if gh release view "$TAG" --repo "$GH_REPO" >/dev/null 2>&1; then
            echo "should_release=false" >> "$GITHUB_OUTPUT"
            echo "Release $TAG already exists — skipping scans"
          else
            echo "should_release=true" >> "$GITHUB_OUTPUT"
            echo "Release $TAG does not exist — proceeding with scans"
          fi
  # ============================================================================
  # SECURITY GATE - All security scans must pass before release proceeds
  # ============================================================================
  # This gate runs all security scans (17 checks): SAST, container/IaC,
  # dependency, secret detection, workflow security, runtime validation,
  # and backwards-compatibility.
  # The build job depends on this gate, ensuring no release can be created
  # without passing all security checks.
  #
  # NOTE: Docker publishing now runs inline (reusable workflow_call to
  # docker-publish.yml), and PyPI publishing is still dispatched via
  # repository_dispatch to publish.yml (PyPI Trusted Publishing does not
  # support reusable workflows — see pypa/gh-action-pypi-publish#166 and
  # pypi/warehouse#11096). Neither publish path is triggered by creating a
  # release via the GitHub UI — only this workflow can dispatch them — so
  # the security-gate flow is preserved.
  # ============================================================================
  release-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    uses: ./.github/workflows/release-gate.yml
    permissions:
      contents: read
      security-events: write
      actions: read
      packages: read         # needed by codeql-scan

  # ============================================================================
  # CI GATE - All CI checks must pass before release proceeds
  # ============================================================================
  # This gate runs all PR-quality checks (linting, type checking, tests,
  # validation) that complement the security-focused release-gate.
  # ============================================================================
  ci-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    uses: ./.github/workflows/ci-gate.yml
    permissions:
      contents: write       # Required: docker-tests deploys coverage to gh-pages (force push)
      pull-requests: write  # Required: docker-tests posts coverage PR comments (no-op in release context)
    secrets:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

  # ============================================================================
  # TEST GATE (Advisory) - WebKit tests run but don't block releases
  # ============================================================================
  # Cosmetic rendering differences in WebKit should not block releases.
  # This gate is advisory: failures are logged but the release proceeds.
  # ============================================================================
  test-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    # Runs in parallel with release-gate (no dependency besides version-check)
    uses: ./.github/workflows/playwright-webkit-tests.yml
    permissions:
      contents: read
      checks: write

  # ============================================================================
  # E2E TEST GATE (Required) - Puppeteer functional tests
  # ============================================================================
  # Verifies UI search workflow via real API calls (OpenRouter + Serper).
  # Must pass before any release proceeds.
  # ============================================================================
  e2e-test-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    # Runs in parallel with release-gate (no dependency besides version-check)
    uses: ./.github/workflows/puppeteer-e2e-tests.yml
    permissions:
      contents: read
      pull-requests: write  # Required: puppeteer-e2e-tests posts test result comments (no-op in release context)
      issues: write         # Required: puppeteer-e2e-tests removes labels on completion (no-op in release context)
    secrets:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}

  # ============================================================================
  # RESPONSIVE TEST GATE (Advisory) - Responsive UI tests across viewports
  # ============================================================================
  # Tests mobile and desktop responsive layouts. Failures are logged but
  # don't block releases (cosmetic viewport differences).
  # ============================================================================
  responsive-test-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    uses: ./.github/workflows/responsive-ui-tests-enhanced.yml
    permissions:
      contents: read

  # ============================================================================
  # BACKWARDS COMPATIBILITY GATE (Required) - Database version compatibility
  # ============================================================================
  # Verifies current code can open databases created by previous PyPI versions.
  # Must pass before any release proceeds.
  # ============================================================================
  compat-test-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    # Runs in parallel with release-gate (no dependency besides version-check)
    uses: ./.github/workflows/backwards-compatibility.yml
    permissions:
      contents: read

  # ============================================================================
  # COMPOSE INTEGRATION GATE (Required) - Bundled docker-compose smoke test
  # ============================================================================
  # Brings up the bundled docker-compose.yml end-to-end and verifies the whole
  # stack (searxng, ollama, local-deep-research) reaches a healthy state.
  # Would have caught #3874 (broken cap_drop on SearXNG) before users hit it.
  # Only runs at release time — not on the daily release-gate cron — because
  # the failure modes are tied to actual compose / image changes that ride
  # along with releases.
  # ============================================================================
  compose-integration-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    uses: ./.github/workflows/compose-integration-test.yml
    permissions:
      contents: read

  # ============================================================================
  # VULTURE DEAD CODE GATE (Advisory) - Detects unreachable / unused code
  # ============================================================================
  # Scans src/ for dead code using vulture.  Results are visible in the
  # Actions tab but failures do NOT block the release.
  # ============================================================================
  vulture-gate:
    needs: [version-check]
    if: needs.version-check.outputs.should_release == 'true'
    uses: ./.github/workflows/vulture-dead-code.yml
    permissions:
      contents: read

  build:
    needs: [version-check, release-gate, ci-gate, test-gate, e2e-test-gate, responsive-test-gate, compat-test-gate, compose-integration-gate, vulture-gate]
    # test-gate (Playwright WebKit), responsive-test-gate, and vulture-gate are advisory
    # release-gate, ci-gate, e2e-test-gate, compat-test-gate, and compose-integration-gate are required
    if: ${{ !cancelled() && needs.release-gate.result == 'success' && needs.ci-gate.result == 'success' && needs.e2e-test-gate.result == 'success' && needs.compat-test-gate.result == 'success' && needs.compose-integration-gate.result == 'success' }}
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.version.outputs.version }}
      tag: ${{ steps.version.outputs.tag }}
      short_sha: ${{ steps.version.outputs.short_sha }}
      release_exists: ${{ steps.check_release.outputs.exists }}
      hashes: ${{ steps.hash.outputs.hashes }}
    permissions:
      contents: read
      pull-requests: read
      id-token: write  # Required for cosign keyless signing with GitHub OIDC

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Verify build commit is on main
        # `push: branches: main` is trivially on main, but `push: tags: v*`
        # and `workflow_dispatch` can fire from any ref — a tag pointing at
        # a feature-branch commit, or a manual dispatch from a stale branch,
        # would otherwise sail through every gate. `fetch-depth: 0` above
        # makes origin/main reachable for the merge-base check.
        run: |
          set -euo pipefail
          if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
            echo "::error::Build commit $GITHUB_SHA is not an ancestor of origin/main — refusing to release"
            exit 1
          fi
          echo "Build commit verified on main: $GITHUB_SHA"

      - name: Determine version
        id: version
        env:
          EVENT_NAME: ${{ github.event_name }}
          GIT_REF: ${{ github.ref }}
        run: |
          if [[ "$EVENT_NAME" = "push" && "$GIT_REF" =~ ^refs/tags/ ]]; then
            # Extract version from tag (remove 'v' prefix)
            VERSION="${GITHUB_REF#refs/tags/v}"
          else
            # Get version from __version__.py (main branch push or manual trigger)
            VERSION=$(grep -oP '(?<=__version__ = ")[^"]*' src/local_deep_research/__version__.py)
          fi
          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
          SHORT_SHA="${GITHUB_SHA:0:7}"
          echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT"

      - name: Check if release already exists
        id: check_release
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          VERSION_TAG: ${{ steps.version.outputs.tag }}
        run: |
          if gh release view "$VERSION_TAG" >/dev/null 2>&1; then
            echo "Release $VERSION_TAG already exists, skipping..."
            echo "exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "Release $VERSION_TAG does not exist, proceeding..."
            echo "exists=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Verify version matches __version__.py
        if: steps.check_release.outputs.exists == 'false'
        env:
          EXPECTED_VERSION: ${{ steps.version.outputs.version }}
        run: |
          VERSION_FILE_VERSION=$(grep -oP '(?<=__version__ = ")[^"]*' src/local_deep_research/__version__.py)
          if [ "$VERSION_FILE_VERSION" != "$EXPECTED_VERSION" ]; then
            echo "Error: Version mismatch!"
            echo "Tag/input version: $EXPECTED_VERSION"
            echo "__version__.py version: $VERSION_FILE_VERSION"
            exit 1
          fi
          echo "Version verified: $EXPECTED_VERSION"

      # SBOM generation uses the SHA-pinned trivy-action with an explicit
      # binary version pin. Previously this step shelled out
      # `apt-get install -y trivy` from the (unpinned) Aqua apt repo, which
      # in a job carrying `id-token: write` (line 229) meant a compromised
      # apt mirror could exfiltrate the OIDC request token AND tamper with
      # the SBOM bytes that the next step (Sign release artifacts) signs
      # under this repo's Sigstore identity — i.e., a fraudulent SBOM
      # carrying a legitimate cosign cert. Pin the action by SHA + the
      # binary by version tag so the toolchain is reproducible and matches
      # docker-publish.yml / prerelease-docker.yml.
      - name: Generate SBOM (SPDX JSON)
        if: steps.check_release.outputs.exists == 'false'
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: 'fs'
          scan-ref: '.'
          format: 'spdx-json'
          output: 'sbom-spdx.json'
          version: 'v0.69.2'

      - name: Generate SBOM (CycloneDX)
        if: steps.check_release.outputs.exists == 'false'
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: 'fs'
          scan-ref: '.'
          format: 'cyclonedx'
          output: 'sbom-cyclonedx.json'
          version: 'v0.69.2'

      - name: List SBOMs
        if: steps.check_release.outputs.exists == 'false'
        run: |
          echo "SBOM files generated:"
          ls -lh sbom-*.json

      - name: Install Cosign
        if: steps.check_release.outputs.exists == 'false'
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
        with:
          # Pin to cosign v2.x — v4.1.2 of the installer ships cosign v3.0.6
          # by default, but v3 enables --new-bundle-format ON BY DEFAULT.
          # That changes the signature/attestation on-wire format and would
          # require all downstream verifiers (including end-users running
          # the README cosign-verify recipe) to also be on v3+. Pin to
          # v2.6.3 (latest v2.x, includes the GHSA-w6c6-c85g-mmv6 patch)
          # for the legacy tag-based sigstore format until we explicitly
          # migrate to v3.
          cosign-release: 'v2.6.3'

      - name: Sign release artifacts with Sigstore
        if: steps.check_release.outputs.exists == 'false'
        run: |
          echo "=== Signing release artifacts with Sigstore ==="

          # Sign SBOM files using keyless signing (OIDC).
          # `--bundle` writes a Sigstore protobuf bundle containing the
          # signature, certificate, and Rekor inclusion proof in one file.
          # Supported in cosign v2.4.0+ (we're pinned to v2.6.3 above).
          cosign sign-blob --yes --bundle sbom-spdx.json.bundle sbom-spdx.json
          cosign sign-blob --yes --bundle sbom-cyclonedx.json.bundle sbom-cyclonedx.json

          echo "Signed SBOM files with Sigstore"
          echo ""
          echo "Bundle files generated (contain signature + certificate):"
          ls -lh sbom-*.bundle

      - name: Verify SBOM bundles and sanity-check SBOM contents
        if: steps.check_release.outputs.exists == 'false'
        env:
          GH_REPO: ${{ github.repository }}
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail

          # Round-trip: prove the README cosign-verify-blob recipe still
          # works for end users. cosign signs above but never verifies, so a
          # sigstore/cert-chain regression would silently break downstream
          # verification — caught only when a user complains.
          echo "=== Verifying SBOM bundles round-trip ==="
          for bundle in sbom-spdx.json.bundle sbom-cyclonedx.json.bundle; do
            json="${bundle%.bundle}"
            cosign verify-blob \
              --bundle "$bundle" \
              --certificate-identity-regexp="^https://github.com/${GH_REPO}/" \
              --certificate-oidc-issuer="https://token.actions.githubusercontent.com" \
              "$json"
          done
          echo "Both SBOM bundles verify cleanly"

          # Trivy can emit a near-empty SBOM on partial scan failures
          # without surfacing a non-zero exit. Set a per-format floor at
          # 80% of the previous stable release's count; fall back to a hard
          # floor of 50 when no prior SBOM is fetchable (first release, GH
          # outage, releases predating SBOM attachment). SPDX and CycloneDX
          # counts differ (virtual packages, metadata-only entries) so each
          # format gets its own floor derived from its own prior count.
          # A cap of 2000 prevents runaway drift if a prior SBOM was
          # anomalously inflated by a Trivy bug.
          echo ""
          echo "=== Sanity-checking SBOM package counts ==="
          SPDX_COUNT=$(jq '.packages | length' sbom-spdx.json)
          CYCDX_COUNT=$(jq '.components | length' sbom-cyclonedx.json)
          echo "Current: SPDX=${SPDX_COUNT}, CycloneDX=${CYCDX_COUNT}"

          HARD_FLOOR=50
          FLOOR_CAP=2000
          SPDX_FLOOR="$HARD_FLOOR"
          CYCDX_FLOOR="$HARD_FLOOR"
          PREV_TAG=$(gh release list --repo "$GH_REPO" --limit 5 \
            --json tagName,isPrerelease,isDraft \
            --jq '[.[] | select(.isPrerelease==false and .isDraft==false)][0].tagName // empty')
          if [[ -n "$PREV_TAG" ]]; then
            PREV_DIR=/tmp/prev-sbom
            rm -rf "$PREV_DIR" && mkdir -p "$PREV_DIR"
            if gh release download "$PREV_TAG" --repo "$GH_REPO" \
                --pattern 'sbom-spdx.json' --pattern 'sbom-cyclonedx.json' \
                --dir "$PREV_DIR" 2>/dev/null; then
              if [[ -f "$PREV_DIR/sbom-spdx.json" ]]; then
                PREV_SPDX=$(jq '.packages | length' "$PREV_DIR/sbom-spdx.json")
                DYN=$(( PREV_SPDX * 80 / 100 ))
                DYN=$(( DYN < FLOOR_CAP ? DYN : FLOOR_CAP ))
                [[ "$DYN" -gt "$HARD_FLOOR" ]] && SPDX_FLOOR="$DYN"
                echo "Prior SPDX: $PREV_SPDX -> floor: $SPDX_FLOOR"
              fi
              if [[ -f "$PREV_DIR/sbom-cyclonedx.json" ]]; then
                PREV_CYCDX=$(jq '.components | length' "$PREV_DIR/sbom-cyclonedx.json")
                DYN=$(( PREV_CYCDX * 80 / 100 ))
                DYN=$(( DYN < FLOOR_CAP ? DYN : FLOOR_CAP ))
                [[ "$DYN" -gt "$HARD_FLOOR" ]] && CYCDX_FLOOR="$DYN"
                echo "Prior CycloneDX: $PREV_CYCDX -> floor: $CYCDX_FLOOR"
              fi
            else
              echo "No SBOMs in $PREV_TAG (older releases predate SBOM attachment) -> hard floor $HARD_FLOOR"
            fi
          else
            echo "No previous stable release found -> hard floor $HARD_FLOOR"
          fi

          if [[ "$SPDX_COUNT" -lt "$SPDX_FLOOR" ]]; then
            echo "::error::SPDX SBOM appears truncated ($SPDX_COUNT < floor $SPDX_FLOOR). Trivy may have failed silently."
            exit 1
          fi
          if [[ "$CYCDX_COUNT" -lt "$CYCDX_FLOOR" ]]; then
            echo "::error::CycloneDX SBOM appears truncated ($CYCDX_COUNT < floor $CYCDX_FLOOR). Trivy may have failed silently."
            exit 1
          fi
          echo "SBOM counts above per-format floors (SPDX≥$SPDX_FLOOR, CycloneDX≥$CYCDX_FLOOR)"

      - name: Generate hashes for SLSA provenance
        if: steps.check_release.outputs.exists == 'false'
        id: hash
        run: |
          # Generate SHA256 hashes for all release artifacts
          # Format: base64(sha256hash filename\nsha256hash filename\n...)
          echo "Generating hashes for SLSA provenance..."
          HASHES=$(sha256sum sbom-spdx.json sbom-spdx.json.bundle sbom-cyclonedx.json sbom-cyclonedx.json.bundle | base64 -w0)
          echo "hashes=$HASHES" >> "$GITHUB_OUTPUT"
          echo "Hashes generated for $(echo "$HASHES" | base64 -d | wc -l) files"

      - name: Upload artifacts for release
        if: steps.check_release.outputs.exists == 'false'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: release-artifacts
          path: |
            sbom-spdx.json
            sbom-spdx.json.bundle
            sbom-cyclonedx.json
            sbom-cyclonedx.json.bundle
          # 7 days, NOT 1: this job runs BEFORE the `release` environment
          # approval, and create-release downloads the artifact AFTER it.
          # The approval is human-paced and unbounded — with retention 1
          # the v1.9.0 artifact expired 24h after build while the approval
          # came ~30h later, failing create-release with "Artifact not
          # found" and leaving the release unrecoverable (the signed SBOM
          # bytes referenced by the SLSA provenance subjects were gone).
          #
          # NOTE: the EFFECTIVE window for approving (or re-running a
          # failed) create-release is 5 days, not 7 — the provenance job
          # (also pre-approval) uploads provenance.intoto.jsonl inside the
          # SLSA generator reusable workflow, which hardcodes
          # retention-days: 5 with no caller input to change it (verified
          # at the pinned v2.1.0 commit). 7 here just keeps THIS artifact
          # from ever being the binding constraint; if the generator gains
          # a retention input, align both values.
          retention-days: 7

  # Generate SLSA provenance for release artifacts
  provenance:
    needs: [build]
    # !cancelled() overrides the implicit success() which fails when advisory test-gate jobs fail
    if: ${{ !cancelled() && needs.build.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    permissions:
      actions: read
      id-token: write
      contents: write
    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@f7dd8c54c2067bafc12ca7a55595d5ee9b75204a # v2.1.0
    with:
      base64-subjects: "${{ needs.build.outputs.hashes }}"
      upload-assets: false  # We'll upload manually with the release
      provenance-name: "provenance.intoto.jsonl"
      compile-generator: true  # Build from source to bypass TUF key validation issues

  # Build, sign, and push the canonical Docker image. This is the single
  # build for the release — publish-docker (the reusable workflow_call
  # to docker-publish.yml) later retags this manifest by digest. The jobs
  # inside this reusable workflow declare `environment: release` so they
  # can read the env-scoped DOCKER_USERNAME / DOCKER_PASSWORD; the FIRST
  # (and only) `release` env approval click in this run unlocks ALL
  # release-env jobs together: prerelease-docker, publish-docker,
  # trigger-pypi, monitor-pypi, and create-release. After the
  # atomicity refactor, docker-publish is inline (reusable workflow_call)
  # so it does not require a separate second approval — its result is
  # visible to downstream jobs (create-release, cleanup-on-rejection) in
  # the same run. (Earlier iterations of the build-once-promote refactor
  # tried to run the canonical build pre-approval to give a real
  # test-then-approve window; that required repo-level Docker Hub
  # secrets, which we deliberately don't have.) The manifest_digest
  # output flows through to the promote step so digest preservation can
  # be verified end-to-end.
  prerelease-docker:
    needs: [build, provenance]
    if: ${{ !cancelled() && needs.build.result == 'success' && needs.provenance.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    uses: ./.github/workflows/prerelease-docker.yml
    with:
      version: ${{ needs.build.outputs.version }}
      short_sha: ${{ needs.build.outputs.short_sha }}
    secrets:
      # Explicit list (not `secrets: inherit`) — defense-in-depth so a
      # future edit to prerelease-docker.yml that references an unrelated
      # secret would fail loudly rather than silently accessing inherited
      # values like PAT_TOKEN, OPENROUTER_API_KEY, or SERPER_API_KEY.
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    permissions:
      # Caller permissions cap what the called workflow can request.
      # Must include id-token: write so cosign keyless OIDC works on the
      # called workflow's create-manifest job.
      # NOTE: no `packages: write` — Docker Hub auth uses DOCKER_PASSWORD
      # secret, not GITHUB_TOKEN. `packages: write` only matters for ghcr.io.
      contents: read
      id-token: write

  # Retag the prerelease manifest as the release tags (:VERSION, :MAJOR_MINOR,
  # :latest). Runs as a reusable workflow_call so its result is visible to
  # downstream jobs in this run — specifically: create-release blocks on
  # success here, and cleanup-on-rejection keys its cosign-deletion logic on
  # `needs.publish-docker.result`. Before this refactor, docker-publish.yml
  # was triggered via repository_dispatch and its outcome was invisible to
  # the parent release.yml run, which made cosign artifact cleanup unsafe
  # after a partial retag failure (release tags could exist sharing the
  # prerelease manifest digest, and deleting `sha256-<digest>.{sig,att}`
  # would invalidate those release-tag signatures).
  publish-docker:
    needs: [build, prerelease-docker]
    if: ${{ !cancelled() && needs.prerelease-docker.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    uses: ./.github/workflows/docker-publish.yml
    with:
      tag: ${{ needs.build.outputs.tag }}
      source_tag: prerelease-v${{ needs.build.outputs.version }}-${{ needs.build.outputs.short_sha }}
      expected_digest: ${{ needs.prerelease-docker.outputs.manifest_digest }}
    secrets:
      # Explicit secret pass — env-scoped (`release`) DOCKER_USERNAME and
      # DOCKER_PASSWORD become available to the callee's `promote` job
      # because that job declares `environment: release`.
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    permissions:
      # cosign VERIFY (the only sigstore operation in the callee) is a
      # read-only check against the public Rekor/Fulcio infrastructure —
      # no GitHub OIDC token is minted, so id-token: write is unnecessary.
      # OIDC is only needed for cosign SIGN, which happens upstream in
      # prerelease-docker.yml.
      contents: read

  create-release:
    # create-release waits for prerelease-docker (canonical image exists,
    # signed and attested), publish-docker (release tags retagged with
    # digest preserved, cosign transitivity verified), and monitor-pypi
    # (PyPI publish has completed). Only then is the GitHub Release
    # published — so the Release page never points at non-existent
    # artifacts. Gated by the `release` environment.
    needs: [build, provenance, prerelease-docker, publish-docker, monitor-pypi]
    if: ${{ !cancelled() && needs.build.result == 'success' && needs.provenance.result == 'success' && needs.prerelease-docker.result == 'success' && needs.publish-docker.result == 'success' && needs.monitor-pypi.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    runs-on: ubuntu-latest
    environment: release
    permissions:
      contents: write
      pull-requests: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Sparse checkout release sources
        # changelog.d/ + pyproject.toml are needed by the "Render changelog"
        # step below (towncrier reads its [tool.towncrier] config from
        # pyproject and consumes fragments from changelog.d/). src/ is
        # deliberately not checked out: the [tool.towncrier] `name` field
        # in pyproject.toml lets towncrier skip its package-import path,
        # and --version on the CLI bypasses package-based version lookup.
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          sparse-checkout: |
            docs/release_notes/
            changelog.d/
            pyproject.toml
          sparse-checkout-cone-mode: false
          fetch-depth: 1
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.12'

      - name: Install towncrier
        run: pip install towncrier==24.8.0

      - name: Render changelog from fragments
        # Produces docs/release_notes/${RELEASE_VERSION}.md from changelog.d/
        # fragments using the towncrier config in pyproject.toml. Towncrier
        # errors on an empty fragment directory, so guard with a glob check —
        # a maintenance release with no fragments should not fail the job.
        # README.md is excluded because changelog.d/README.md is the
        # contributor guide, not a fragment.
        # Fragments are consumed (deleted) locally on the runner; the
        # cleanup-changelog job re-runs this and opens a PR to persist the
        # changes back to main.
        env:
          RELEASE_VERSION: ${{ needs.build.outputs.version }}
        run: |
          set -euo pipefail
          fragment_count=$(find changelog.d -maxdepth 1 -name '*.md' ! -name 'README.md' | wc -l)
          if [[ "$fragment_count" -gt 0 ]]; then
            towncrier build --yes --version "$RELEASE_VERSION"
            echo "Rendered docs/release_notes/${RELEASE_VERSION}.md from ${fragment_count} fragment(s)"
          else
            echo "No changelog fragments found — skipping towncrier build"
          fi

      - name: Download build artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: release-artifacts

      - name: Download SLSA provenance
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: provenance.intoto.jsonl

      - name: Download container-image SBOMs
        # Produced by prerelease-docker.yml — one SBOM per architecture
        # (sbom-amd64.spdx.json, sbom-arm64.spdx.json) from Syft. The
        # cosign attestations on each per-arch digest are the
        # cryptographically authoritative copies; attaching the raw JSON
        # to the GitHub Release makes them discoverable to humans
        # browsing the release page.
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: sbom
          path: container-sbom

      - name: Stage container-image SBOMs for release attachment
        # Rename per-arch SBOMs to avoid collision with the filesystem SBOM
        # `sbom-spdx.json` produced by the build job (Trivy scan of source
        # tree). Container SBOMs are the Syft scan of built image layers
        # per platform — different content.
        run: |
          set -euo pipefail
          # Per-arch SBOMs: rename sbom-amd64.spdx.json → sbom-container-amd64.spdx.json
          # (and similarly for arm64). prerelease-docker.yml no longer
          # produces a manifest-level sbom.spdx.json.
          for f in container-sbom/sbom-*.spdx.json; do
            [ -f "$f" ] || continue
            base=$(basename "$f")
            mv "$f" "sbom-container-${base#sbom-}"
          done
          ls -la sbom-container*.spdx.json || echo "No container SBOMs found"

      - name: List artifacts
        run: |
          echo "Release artifacts:"
          ls -la

      - name: Create GitHub Release
        run: |
          # Compose the release body before creation so the published body is
          # final on first publish (no edit-after window where downstream
          # workflows could see partial notes).
          #
          # Body layout (each section is omitted if its source is empty):
          #   1. AI-generated narrative (model = vars.AI_MODEL). Fed the
          #      hand-written notes, the auto PR list, every PR's title+body,
          #      and the actual diff vs the previous release.
          #   2. Hand-written narrative — docs/release_notes/<version>.md,
          #      rendered earlier in this job by towncrier from
          #      changelog.d/ fragments.
          #   3. Horizontal rule
          #   4. Auto-generated, label-categorized PR list (already starts
          #      with "## What's Changed", emitted by GitHub's generate-notes
          #      API driven by .github/release.yml)
          #
          # The AI step is best-effort: if OPENROUTER_API_KEY is unset or the
          # call fails, we fall through silently with no AI section. Releases
          # must never fail because of an LLM hiccup.
          set -euo pipefail

          # Defense-in-depth: RELEASE_VERSION is interpolated into a regex
          # below, so reject anything that isn't a bare semver string. The
          # build job's "Determine version" step strips `refs/tags/v` for
          # tag pushes and reads __version__.py (bare "1.6.7") otherwise,
          # so a leading `v` here would mean an upstream contract change
          # and should fail loudly.
          if [[ ! "$RELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][a-zA-Z0-9.+_-]+)?$ ]]; then
            echo "ERROR: RELEASE_VERSION '$RELEASE_VERSION' is not a valid bare semver string" >&2
            exit 1
          fi

          RELEASE_NOTES_FILE="docs/release_notes/${RELEASE_VERSION}.md"
          NOTES_FILE="$(mktemp)"
          PROMPT_FILE="$(mktemp)"
          AI_RESPONSE_FILE="$(mktemp)"
          AI_SUMMARY_FILE="$(mktemp)"
          PR_BODIES_FILE="$(mktemp)"
          DIFF_FILE="$(mktemp)"
          # ${DIFF_FILE}.clean is the staging path for the NUL-strip step;
          # rm -f tolerates the case where it doesn't exist.
          trap 'rm -f "$NOTES_FILE" "$PROMPT_FILE" "$AI_RESPONSE_FILE" "$AI_SUMMARY_FILE" "$PR_BODIES_FILE" "$DIFF_FILE" "${DIFF_FILE}.clean"' EXIT

          # `set -e` does NOT exit on a failing command substitution in an
          # assignment, so wrap with explicit failure handling.
          # target_commitish: for a tag that does not exist yet (the normal
          # case), generate-notes otherwise resolves the commit range
          # against default-branch HEAD at call time — after a delayed
          # approval the auto PR list would include PRs merged AFTER the
          # gated commit (the notes-side sibling of the dispatch-time-HEAD
          # drift fixed in this change). Pinning it keeps the PR list
          # consistent with the wheel/image/tag. Ignored when the tag
          # already exists.
          if ! AUTO_BODY=$(gh api -X POST \
              "/repos/${GITHUB_REPOSITORY}/releases/generate-notes" \
              -f tag_name="${RELEASE_TAG}" \
              -f target_commitish="${GITHUB_SHA}" \
              -q .body); then
            echo "ERROR: gh api generate-notes failed for $RELEASE_TAG" >&2
            exit 1
          fi

          # The hand-written narrative is the per-version file produced by
          # the "Render changelog from fragments" step above. Empty if there
          # were no fragments in changelog.d/ for this release (towncrier is
          # skipped in that case) — release then proceeds with auto-notes
          # plus whatever the AI produces from PR bodies + diff.
          HAND_NOTES=""
          if [[ -f "$RELEASE_NOTES_FILE" ]]; then
            HAND_NOTES=$(cat "$RELEASE_NOTES_FILE")
          else
            echo "WARNING: $RELEASE_NOTES_FILE not found — release proceeds without hand-written notes"
          fi

          # ---- PR descriptions (batch via GraphQL, one round-trip) ----
          # Extract PR numbers from the auto-body's pull/<num> URLs. Dedupe
          # in case the same PR is linked twice. GraphQL aliases let us fetch
          # all PR titles + bodies in a single request, sidestepping the
          # rate-limit cost of N sequential REST calls.
          PR_NUMBERS=$(printf '%s' "$AUTO_BODY" | grep -oE 'pull/[0-9]+' | grep -oE '[0-9]+' | sort -u || true)
          : > "$PR_BODIES_FILE"
          if [[ -n "$PR_NUMBERS" ]]; then
            # GraphQL has a node/complexity ceiling. One aliased
            # pullRequest(number:) lookup per PR with three shallow fields
            # is cheap, but unbounded fan-out can still trip the limit on a
            # huge release. Cap at 200 PRs per query — sized for LDR
            # releases that bundle dependency-bump traffic (v1.6.10 had
            # 144 unique PRs). The 750k char overall prompt cap protects
            # against the per-PR body fan-out blowing the context window.
            pr_count=$(printf '%s\n' "$PR_NUMBERS" | grep -c . || true)
            if [[ "$pr_count" -gt 200 ]]; then
              echo "WARNING: $pr_count PRs in release — capping to first 200 for GraphQL fetch"
              PR_NUMBERS=$(printf '%s\n' "$PR_NUMBERS" | head -200)
            fi
            OWNER="${GITHUB_REPOSITORY%/*}"
            REPO="${GITHUB_REPOSITORY#*/}"
            QUERY="query { repository(owner: \"$OWNER\", name: \"$REPO\") {"
            i=0
            while IFS= read -r num; do
              [[ -z "$num" ]] && continue
              QUERY+=" pr${i}: pullRequest(number: ${num}) { number title body }"
              i=$((i + 1))
            done <<< "$PR_NUMBERS"
            QUERY+=" } }"

            if gh api graphql -f query="$QUERY" \
              --jq '.data.repository | to_entries[] | .value | select(. != null) | "### PR #\(.number): \(.title)\n\((.body // "") | .[0:2000])\n"' \
              > "$PR_BODIES_FILE" 2>/dev/null; then
              echo "Fetched $(printf '%s\n' "$PR_NUMBERS" | wc -l) PR descriptions ($(wc -c < "$PR_BODIES_FILE") chars)"
            else
              echo "WARNING: GraphQL PR fetch failed — proceeding without PR descriptions"
              : > "$PR_BODIES_FILE"
            fi
          fi

          # ---- Diff between previous release and current ----
          # /releases/latest excludes drafts and prereleases. Returns 404 if
          # there is no prior stable release — the AI then runs without a
          # diff input. The diff is produced LOCALLY by git (shallow-fetch
          # of the previous tag), not by the compare API: the API silently
          # caps .files[] at 300 entries (the v1.8.1→v1.9.0 compare hit
          # exactly 300) and lists files alphabetically, so tail-truncation
          # dropped patches arbitrarily instead of by importance. The local
          # diff is complete, starts with a full --stat, and orders patch
          # bodies by signal (src/ first, largest change first) so the
          # 700,000-char budget (~175k tokens, headroom in Kimi K2
          # Thinking's 262k context for prompt + 64k output) trims the
          # least informative patches first. Same noise filter as before
          # (lockfiles, generated docs, SBOM, static assets, snapshots,
          # binaries).
          : > "$DIFF_FILE"
          PREV_TAG=""
          if PREV_TAG=$(gh api "/repos/${GITHUB_REPOSITORY}/releases/latest" --jq '.tag_name' 2>/dev/null) && [[ -n "$PREV_TAG" && "$PREV_TAG" != "null" ]]; then
            # On a workflow re-run after the release already exists,
            # /releases/latest returns the just-created tag. Fall back to
            # the second-most-recent stable release so the diff is
            # meaningful instead of empty.
            if [[ "$PREV_TAG" == "$RELEASE_TAG" ]]; then
              echo "NOTE: /releases/latest is the current tag (re-run scenario) — looking up previous"
              # Pass RELEASE_TAG as a jq variable rather than splicing it
              # into the program text. RELEASE_TAG is already validated as
              # bare semver above, but --arg keeps shell quoting and jq
              # quoting fully separated regardless.
              RELEASES_JSON=$(gh api "/repos/${GITHUB_REPOSITORY}/releases?per_page=10" 2>/dev/null || printf '[]')
              PREV_TAG=$(printf '%s' "$RELEASES_JSON" | jq -r --arg rel "$RELEASE_TAG" \
                '[.[] | select(.draft == false and .prerelease == false and .tag_name != $rel)][0].tag_name // empty' \
                2>/dev/null || true)
              if [[ -z "$PREV_TAG" ]]; then
                echo "WARNING: could not find a prior stable release — skipping diff"
              fi
            fi
          fi
          if [[ -n "$PREV_TAG" ]]; then
            echo "Fetching previous release tag $PREV_TAG for local diff"
            # Shallow-fetch just the previous tag's commit. NOTE:
            # actions/checkout configures a blob:none partial clone whenever
            # sparse-checkout is set, and this fetch inherits that filter —
            # the object store has the TREES of both commits but not the
            # blobs. The diffs below still work because git lazily
            # batch-fetches missing blobs from the promisor remote over
            # anonymous HTTPS (public repo; persist-credentials: false is
            # irrelevant to an unauthenticated fetch). If that lazy fetch
            # ever breaks (repo made private, anonymous throttling), the
            # python builder's git call exits non-zero and the guarded
            # `if !` below degrades to WARNING + empty diff — the release
            # itself is never blocked by this step.
            if git fetch --quiet --depth=1 origin "refs/tags/${PREV_TAG}:refs/tags/${PREV_TAG}"; then
              # Best-effort like every other AI input: a diff failure must
              # not fail the release, only shrink the prompt.
              if ! python3 - "$PREV_TAG" "$GITHUB_SHA" "$DIFF_FILE" 700000 <<'PYEOF'
          import re
          import subprocess
          import sys

          prev, head, out_path, budget = (
              sys.argv[1],
              sys.argv[2],
              sys.argv[3],
              int(sys.argv[4]),
          )

          # Same noise filter the old compare-API jq expression used.
          NOISE = re.compile(
              r"(package-lock\.json$|pdm\.lock$|\.min\.(js|css)$"
              r"|^docs/CONFIGURATION\.md$|^docs/ci/workflow-status\.md$"
              r"|^sbom-|/__pycache__/|\.pyc$|^docs/images/"
              r"|^src/local_deep_research/web/static/"
              r"|/(golden_master|snapshot)|\.snap$)"
          )


          def git(*args):
              # errors="replace" keeps undecodable bytes from raising;
              # --no-renames keeps numstat paths plain (no "{a => b}"
              # munging) so the per-file diff below can address them.
              return subprocess.run(
                  ["git", "--no-pager", *args],
                  check=True,
                  capture_output=True,
                  text=True,
                  errors="replace",
              ).stdout


          rng = f"{prev}..{head}"
          # The complete --stat goes first and is never truncated: even
          # when patch bodies below are dropped for budget, the model
          # always sees the full shape of the release.
          stat = git("diff", "--no-renames", "--stat=200", rng)

          # -z terminates entries with NUL and, crucially, disables
          # core.quotePath C-quoting: without it a path with non-ASCII or
          # special characters comes back as a quoted literal ("a\303\251")
          # that the per-file pathspec below silently fails to match —
          # the file would be counted as included while its patch is empty.
          files = []
          for entry in git("diff", "--no-renames", "--numstat", "-z", rng).split("\0"):
              if not entry:
                  continue
              adds, dels, path = entry.split("\t", 2)
              if adds == "-" or NOISE.search(path):  # binary or noise
                  continue
              files.append((path, int(adds) + int(dels)))


          # Signal order: source first, then everything else, then
          # tests/CI/docs; biggest change first within each group. Budget
          # overflow then drops the LEAST informative patches instead of
          # whatever sorted last alphabetically.
          def group(path):
              if path.startswith("src/"):
                  return 0
              if path.startswith(("tests/", ".github/", "docs/")):
                  return 2
              return 1


          files.sort(key=lambda f: (group(f[0]), -f[1]))

          chunks = [f"===== DIFFSTAT ({prev}...{head}, complete) =====\n{stat}\n"]
          written = len(chunks[0])
          dropped = []
          unmatched = []
          for path, _ in files:
              # :(literal) keeps glob metacharacters in a filename ([ * ?)
              # from being interpreted as pathspec wildcards.
              patch = git("diff", "--no-renames", rng, "--", f":(literal){path}")
              if not patch:
                  # A pathspec that matches nothing exits 0 with empty
                  # output — e.g. a filename whose bytes don't round-trip
                  # the errors="replace" UTF-8 decode (U+FFFD in the
                  # decoded path no longer equals the on-disk bytes).
                  # Without this branch the file would be COUNTED as
                  # included while its patch is silently absent.
                  unmatched.append(path)
                  continue
              if written + len(patch) > budget:
                  dropped.append(path)
                  continue
              chunks.append(patch)
              written += len(patch)

          if dropped:
              shown = ", ".join(dropped[:20]) + (" ..." if len(dropped) > 20 else "")
              chunks.append(
                  f"\n[... {len(dropped)} lower-priority file diffs omitted to fit "
                  f"the {budget}-char budget: {shown} ...]\n"
              )
          if unmatched:
              shown = ", ".join(unmatched[:10]) + (" ..." if len(unmatched) > 10 else "")
              chunks.append(
                  f"\n[... {len(unmatched)} file diffs unavailable (pathspec did not "
                  f"match after decoding): {shown} ...]\n"
              )
          with open(out_path, "w", encoding="utf-8") as fh:
              fh.write("".join(chunks))
          print(
              f"Diff input: {len(files) - len(dropped) - len(unmatched)}/{len(files)} "
              f"file patches + complete diffstat, {written} chars"
              + (f"; {len(unmatched)} unmatched pathspec(s): {unmatched[:10]}" if unmatched else ""),
              file=sys.stderr,
          )
          PYEOF
              then
                echo "WARNING: local git diff failed — proceeding without diff"
                : > "$DIFF_FILE"
              fi
              # NUL bytes break jq --rawfile JSON encoding; strip them.
              tr -d '\000' < "$DIFF_FILE" > "${DIFF_FILE}.clean" && mv "${DIFF_FILE}.clean" "$DIFF_FILE"
              echo "Diff size: $(wc -c < "$DIFF_FILE") bytes"
            else
              echo "WARNING: could not fetch tag ${PREV_TAG} — proceeding without diff"
            fi
          else
            echo "No previous stable release found — skipping diff"
          fi

          # ---- AI narrative (best-effort) -------------------------------------
          AI_SUMMARY=""
          if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then
            echo "OPENROUTER_API_KEY not set — skipping AI summary"
          else
            # Compose the prompt: instructions (from $SUMMARY_PROMPT env)
            # followed by labelled input sections. Order matters for
            # truncation safety — diff is last because it's the largest
            # variable input and gets trimmed first if the overall prompt
            # exceeds the 750k char safety cap.
            {
              printf '%s\n' "$SUMMARY_PROMPT"
              printf '\n----- HAND-WRITTEN NOTES -----\n'
              if [[ -n "$HAND_NOTES" ]]; then
                printf '%s\n' "$HAND_NOTES"
              else
                printf '(none)\n'
              fi
              printf '\n----- AUTO-GENERATED PR LIST -----\n%s\n' "$AUTO_BODY"
              if [[ -s "$PR_BODIES_FILE" ]]; then
                printf '\n----- PR DESCRIPTIONS -----\n'
                cat "$PR_BODIES_FILE"
              fi
              if [[ -s "$DIFF_FILE" ]]; then
                printf '\n----- DIFF (%s...%s) -----\n' "$PREV_TAG" "$RELEASE_TAG"
                cat "$DIFF_FILE"
              fi
            } > "$PROMPT_FILE"

            # Overall prompt safety cap. Kimi K2 Thinking has a 262k token
            # context; leaving 64k for max_tokens output means ~198k for input,
            # which at ~3 chars/token for code-heavy diff is ~600k chars. We
            # cap at 750k chars to absorb the per-char/token variance — if
            # PR descriptions are unusually long alongside a near-cap diff,
            # the tail (diff) gets trimmed first since it sits last.
            # Character-aware truncation (Python) — see diff section above.
            python3 - "$PROMPT_FILE" 750000 'prompt truncated at 750,000 chars' <<'PYEOF'
          import sys
          path, max_chars, marker = sys.argv[1], int(sys.argv[2]), sys.argv[3]
          with open(path, encoding='utf-8', errors='replace') as f:
              content = f.read()
          if len(content) > max_chars:
              content = content[:max_chars] + f"\n[... {marker} ...]\n"
              with open(path, 'w', encoding='utf-8') as f:
                  f.write(content)
              print(f"Truncated to {len(content)} chars", file=sys.stderr)
          PYEOF

            # reasoning.max_tokens caps the thinking trace so the model
            # cannot burn the whole max_tokens budget on reasoning and emit
            # empty content. This was the root cause of the v1.6.10 release
            # AI step failing with finish_reason: length and empty .content.
            JSON_PAYLOAD=$(jq -n \
              --arg model "$AI_MODEL" \
              --rawfile content "$PROMPT_FILE" \
              --argjson temperature "$AI_TEMPERATURE" \
              --argjson max_tokens "$AI_MAX_TOKENS" \
              --argjson reasoning_max_tokens "$AI_REASONING_MAX_TOKENS" \
              '{model:$model, messages:[{role:"user", content:$content}], temperature:$temperature, max_tokens:$max_tokens, reasoning:{max_tokens:$reasoning_max_tokens}}')

            # Up to 2 attempts (1 retry) with a short backoff. The model
            # endpoint can hiccup on cold starts or transient rate-limits;
            # one retry covers nearly all of those without delaying releases.
            # Default --max-time is 900s (15 min) — kimi-k2-thinking and
            # other thinking models can spend many minutes reasoning before
            # producing output, so the previous 120s ceiling was too
            # tight (observed on the v1.6.8 release run).
            HTTP_CODE="000"
            for attempt in 1 2; do
              HTTP_CODE=$(printf '%s' "$JSON_PAYLOAD" | curl -sS \
                -o "$AI_RESPONSE_FILE" \
                -w '%{http_code}' \
                --max-time "$AI_MAX_TIME" \
                -X POST "https://openrouter.ai/api/v1/chat/completions" \
                -H "Content-Type: application/json" \
                -H "Authorization: Bearer $OPENROUTER_API_KEY" \
                -H "HTTP-Referer: https://github.com/${GITHUB_REPOSITORY}" \
                --data-binary @- || echo "000")
              if [[ "$HTTP_CODE" =~ ^2 ]]; then
                break
              fi
              echo "WARNING: AI summary attempt $attempt failed (HTTP $HTTP_CODE)"
              if [[ "$attempt" -eq 1 ]]; then
                echo "Retrying in 5s..."
                sleep 5
              fi
            done

            if [[ "$HTTP_CODE" =~ ^2 ]]; then
              # OpenRouter normalizes Moonshot's thinking trace to .reasoning
              # (not .reasoning_content — that key is Moonshot's native API
              # shape, exposed only on direct calls). Try .content first;
              # fall back to .reasoning_content then .reasoning. The triple
              # fallback is defense-in-depth: provider normalization is in
              # flux and either variant may surface in the future.
              AI_SUMMARY=$(jq -r '.choices[0].message.content // empty' "$AI_RESPONSE_FILE" 2>/dev/null || true)
              if [[ -z "$AI_SUMMARY" ]]; then
                AI_SUMMARY=$(jq -r '.choices[0].message.reasoning_content // empty' "$AI_RESPONSE_FILE" 2>/dev/null || true)
                [[ -n "$AI_SUMMARY" ]] && echo "NOTE: empty .content; using .reasoning_content from $AI_MODEL"
              fi
              if [[ -z "$AI_SUMMARY" ]]; then
                AI_SUMMARY=$(jq -r '.choices[0].message.reasoning // empty' "$AI_RESPONSE_FILE" 2>/dev/null || true)
                [[ -n "$AI_SUMMARY" ]] && echo "NOTE: empty .content/.reasoning_content; using .reasoning from $AI_MODEL"
              fi
              if [[ -n "$AI_SUMMARY" ]]; then
                echo "AI summary generated (model=$AI_MODEL, ${#AI_SUMMARY} chars)"
                # Hallucination guard: every reference the model emitted
                # (#NNNN, bare pull/issues URL, GH-NNNN — all forms GitHub
                # renders as a reference link) must appear in the INPUT
                # sections of the prompt it was given.
                # The prompt merely INSTRUCTS the model never to invent PR
                # numbers; instructions are not enforcement, and this text
                # publishes sight-unseen. On any unknown reference, drop
                # the whole AI section loudly — the release proceeds with
                # towncrier notes + the auto PR list, the same degrade
                # path as a missing OPENROUTER_API_KEY. The check reads
                # $PROMPT_FILE post-truncation, i.e. exactly what the
                # model saw. It cannot catch invented BEHAVIOR claims,
                # only invented references — but those are the most
                # embarrassing and the only mechanically detectable kind.
                # All-numeric hex colors ("#123456") can false-positive;
                # that costs us the AI section, never a wrong note.
                printf '%s' "$AI_SUMMARY" > "$AI_SUMMARY_FILE"
                if ! python3 - "$PROMPT_FILE" "$AI_SUMMARY_FILE" <<'PYEOF'
          import os
          import re
          import sys

          with open(sys.argv[1], encoding="utf-8", errors="replace") as f:
              prompt = f.read()
          with open(sys.argv[2], encoding="utf-8", errors="replace") as f:
              summary = f.read()

          # Only the INPUT sections feed the allowed set. The instruction
          # text above the first "----- " header contains a literal
          # format example ("Reference PRs as `#1234`") that would
          # otherwise self-whitelist #1234 — a real PR in this repo — for
          # the single most predictable parroting artifact.
          cut = prompt.find("\n----- ")
          inputs = prompt[cut:] if cut != -1 else prompt

          # Both sides scan the same two LOCAL reference forms:
          # - "#NNNN" with a (?<![\w/]) lookbehind, so the cross-repo
          #   shorthand "owner/repo#166" (rendered by GitHub as a link to
          #   the OTHER repo) neither whitelists a local number on the
          #   input side nor counts as a local claim on the output side.
          #   The lookbehind also skips inline anchors whose "#" follows
          #   a word char or slash ("page#4-setup", "docs/#4").
          # - pull/issues URLs scoped to THIS repository — a cross-repo
          #   URL in a PR body must not whitelist its number locally, and
          #   the model repeating one must not trip the guard.
          # GITHUB_REPOSITORY is always set in Actions; if it were ever
          # missing, the KeyError exits non-zero and the AI section is
          # dropped (safe direction).
          repo = re.escape(os.environ["GITHUB_REPOSITORY"])
          allowed = set(re.findall(r"(?<![\w/])#(\d+)\b", inputs))
          allowed |= set(re.findall(repo + r"/(?:pull|issues)/(\d+)\b", inputs))

          # Emitted references: capture local URL forms first, then strip
          # URLs from the summary so their numeric fragment anchors
          # ("https://ex.com/p#4-setup") can't count as prose references,
          # then scan the remaining prose for "#NNNN" and "GH-NNNN".
          # (Round 1 used a (?!-) lookahead against anchors, but that
          # silently removed any "#9999-suffixed" reference from
          # detection entirely — a false-NEGATIVE hole, the wrong
          # direction for a guard. URL-stripping kills the anchor
          # false-positive without blinding the scan.)
          url_refs = set(re.findall(repo + r"/(?:pull|issues)/(\d+)\b", summary))
          prose = re.sub(r"https?://\S+", " ", summary)
          emitted = set(re.findall(r"(?<![\w/])#(\d+)\b", prose))
          emitted |= url_refs
          emitted |= set(re.findall(r"\bGH-(\d+)\b", prose))

          unknown = sorted(emitted - allowed, key=int)
          if unknown:
              print(
                  "AI summary references numbers absent from its input: "
                  + ", ".join("#" + n for n in unknown),
                  file=sys.stderr,
              )
              sys.exit(1)
          print(
              f"AI reference check passed: {len(emitted)} reference(s), "
              "all present in the prompt",
              file=sys.stderr,
          )
          PYEOF
                then
                  echo "WARNING: AI summary contains references not present in its input — dropping AI section (release proceeds with towncrier notes + PR list)"
                  AI_SUMMARY=""
                fi
              else
                echo "WARNING: AI response 2xx but no usable content field — skipping summary"
                # Diagnostic: dump the response shape so the next failure
                # can be debugged without rerunning. Bounded to keep noise
                # out of logs.
                echo "  response keys: $(jq -r 'keys | join(", ")' "$AI_RESPONSE_FILE" 2>/dev/null || echo '<unparseable>')"
                echo "  message keys:  $(jq -r '.choices[0].message | keys | join(", ")' "$AI_RESPONSE_FILE" 2>/dev/null || echo '<missing>')"
                echo "  finish_reason: $(jq -r '.choices[0].finish_reason // "<none>"' "$AI_RESPONSE_FILE" 2>/dev/null)"
                echo "  error field:   $(jq -r '.error // empty' "$AI_RESPONSE_FILE" 2>/dev/null)"
              fi
            else
              echo "WARNING: AI summary call failed (HTTP $HTTP_CODE) — skipping summary"
            fi
          fi
          # ---- end AI narrative -----------------------------------------------

          {
            if [[ -n "$AI_SUMMARY" ]]; then
              printf '%s\n\n' "$AI_SUMMARY"
            fi
            if [[ -n "$HAND_NOTES" ]]; then
              printf '%s\n\n' "$HAND_NOTES"
            fi
            if [[ -n "$AI_SUMMARY" || -n "$HAND_NOTES" ]]; then
              printf -- '---\n\n'
            fi
            printf '%s\n' "$AUTO_BODY"
          } > "$NOTES_FILE"

          # GitHub release body limit is 125,000 chars — the API rejects
          # longer bodies with HTTP 422 and gh CLI does not pre-validate.
          # Truncate from the tail (the auto PR list is the largest variable
          # section in big releases) with a visible marker. Character-aware
          # truncation (Python) — see diff section above.
          python3 - "$NOTES_FILE" 124400 "[notes truncated to fit GitHub's 125,000-char body limit]" <<'PYEOF'
          import sys
          path, max_chars, marker = sys.argv[1], int(sys.argv[2]), sys.argv[3]
          with open(path, encoding='utf-8', errors='replace') as f:
              content = f.read()
          if len(content) > max_chars:
              content = content[:max_chars] + f"\n\n_{marker}_\n"
              with open(path, 'w', encoding='utf-8') as f:
                  f.write(content)
              print(f"Release body truncated to {len(content)} chars", file=sys.stderr)
          PYEOF

          # Includes filesystem SBOM (Trivy scan of source tree) AND
          # container-image SBOM (Syft scan of built image layers, per-arch),
          # Sigstore signatures, and SLSA provenance. The two SBOM kinds
          # describe different surfaces — both are useful for downstream
          # supply-chain consumers.
          # `find ... -print0 | xargs -0 ...` handles the variable number
          # of per-arch container SBOMs gracefully (zero or more).
          set -euo pipefail
          CONTAINER_SBOMS=()
          while IFS= read -r -d '' f; do
            CONTAINER_SBOMS+=("$f")
          done < <(find . -maxdepth 1 -name 'sbom-container*.spdx.json' -print0)
          # --target pins the git tag this call MINTS to the gated release
          # commit. Without it, the Releases API tags default-branch HEAD
          # at create-release time — which, after a human-paced approval
          # delay, is a LATER, un-gated commit: the tag-side twin of the
          # PyPI dispatch-time-HEAD drift fixed in this same change (on
          # v1.9.0 that would have tagged a commit 4 PRs past the one the
          # gates ran against). Ignored when the tag already exists (e.g.
          # tag-push-triggered runs), so it is safe in every trigger mode.
          gh release create "$RELEASE_TAG" \
            --target "$GITHUB_SHA" \
            --repo "$GITHUB_REPOSITORY" \
            --title "Release $RELEASE_VERSION" \
            --notes-file "$NOTES_FILE" \
            sbom-spdx.json \
            sbom-spdx.json.bundle \
            sbom-cyclonedx.json \
            sbom-cyclonedx.json.bundle \
            provenance.intoto.jsonl \
            "${CONTAINER_SBOMS[@]}"
        env:
          # PAT_TOKEN is required here because GITHUB_TOKEN cannot trigger
          # workflows that listen on `release:` (backwards-compatibility.yml,
          # sbom.yml) — that's a documented GITHUB_TOKEN limitation.
          # Minimum scope: `repo` (full scope; `public_repo` is NOT sufficient
          # for the release-creation API on private repos and `repo` works
          # uniformly). The `workflow` scope is NOT needed — it only governs
          # editing files under .github/workflows/ via the API, which this
          # step does not do.
          GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }}
          RELEASE_TAG: ${{ needs.build.outputs.tag }}
          RELEASE_VERSION: ${{ needs.build.outputs.version }}
          # Optional AI summarization. If OPENROUTER_API_KEY is not configured
          # in the release environment, the step skips the AI section silently.
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
          AI_MODEL: ${{ vars.AI_MODEL || 'moonshotai/kimi-k2-thinking' }}
          AI_TEMPERATURE: ${{ vars.AI_RELEASE_SUMMARY_TEMPERATURE || '0.3' }}
          # 64000 matches the AI code-reviewer's working budget. The previous
          # 4000-token cap was the root cause of the v1.6.10 release AI failure:
          # kimi-k2-thinking burned the whole budget on reasoning tokens (which
          # count against max_tokens) and emitted finish_reason=length with an
          # empty .content. Reasoning tokens are pricey ($2.50/M output rate),
          # so reasoning.max_tokens is capped separately below.
          AI_MAX_TOKENS: ${{ vars.AI_RELEASE_SUMMARY_MAX_TOKENS || '64000' }}
          # Caps the thinking trace so the model cannot exhaust max_tokens
          # before emitting visible content. 16000 is well above Kimi's typical
          # 5k-15k reasoning footprint for summarization without giving it
          # room to spiral on a large diff.
          AI_REASONING_MAX_TOKENS: ${{ vars.AI_RELEASE_SUMMARY_REASONING_MAX_TOKENS || '16000' }}
          # Curl --max-time for the OpenRouter call. Default 900s (15 min) —
          # thinking models need substantial headroom on a multi-PR release
          # prompt (the previous 120s default timed out on v1.6.8 with
          # kimi-k2-thinking). Override via vars.AI_RELEASE_SUMMARY_MAX_TIME
          # if you want the AI step to fail faster on stuck calls.
          AI_MAX_TIME: ${{ vars.AI_RELEASE_SUMMARY_MAX_TIME || '900' }}
          SUMMARY_PROMPT: >
            You are writing the top section of a GitHub release page for an open-source
            project. The audience is end users of the software who want to know what
            changed and whether this release matters to them.


            Several inputs follow, each introduced by a header line of the form
            `----- HAND-WRITTEN NOTES -----`, `----- AUTO-GENERATED PR LIST -----`,
            `----- PR DESCRIPTIONS -----`, `----- DIFF (oldtag...newtag) -----`:

            - HAND-WRITTEN NOTES: a maintainer-curated changelog rendered from per-PR
              fragments. May be empty for maintenance releases. When present, these are
              the most authoritative description of intent. They appear verbatim below
              your text in the published release page — do not repeat them line-by-line.

            - AUTO-GENERATED PR LIST: GitHub's label-categorized list of merged PRs for
              this release. Useful for completeness; weak signal for intent. This list
              also appears verbatim below your text — do not repeat it.

            - PR DESCRIPTIONS: the body text of each merged PR. Often explains the why
              behind a change better than the title.

            - DIFF: the actual code change between the previous release and this one.
              Starts with a complete diffstat of every changed file, followed by
              per-file patches ordered by importance (source code first, largest
              changes first); low-priority patches may be omitted to fit, but the
              diffstat is always complete. Use to disambiguate what a PR actually did
              or to catch user-visible changes not well described in PR text.


            Write a helpful narrative that goes at the top of the release page. Lead
            with the most user-visible changes. Flag breaking changes early with a brief
            "what to do" if applicable. Group related changes when natural. Skip
            dependency bumps, internal CI, refactors, and docs-only changes unless
            they're load-bearing for users.


            Size the writeup to the material. A small maintenance release might get one
            short paragraph; a substantial release with new features and breaking
            changes might warrant several paragraphs with subsections. Don't pad. Don't
            open with a meta-preamble like "this release contains X PRs across Y
            categories" — just describe what happened.


            Output GitHub-flavored markdown. Open with a `##` heading naming the theme
            of the release (e.g. `## Friendlier OpenAI-compatible error messages and
            multi-migration upgrade fixes`, or for small releases simply `## What's
            new`). Use `###` for any subsections. Do not emit `#` (the release title
            above is the H1). Reference PRs as `#1234` only when their numbers appear
            in the inputs — never invent PR numbers, features, or behavior. If the
            hand-written notes contradict the PR list or diff, trust the hand-written
            notes; they are the authoritative human narrative.


            No preamble, no closing meta-commentary. Just the section.

  # Persists the towncrier render back to main. The create-release job runs
  # `towncrier build` in a throwaway sparse checkout to produce the rendered
  # release notes for the GitHub release body, but those changes (consumed
  # fragments + new docs/release_notes/<version>.md) never reach the repo
  # from there. Without this job, the next release would re-render all the
  # same fragments. Branch protection on main has 0 required reviews and
  # 0 required status checks, so the maintainer can squash-merge this PR
  # without ceremony.
  cleanup-changelog:
    needs: [build, create-release]
    if: ${{ !cancelled() && needs.create-release.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout the release commit
        # Pinning to ${{ github.sha }} (not `ref: main`) avoids a race:
        # if a new PR with new fragments merges into main between
        # create-release and cleanup-changelog, `ref: main` would consume
        # those new fragments into THIS release's docs/release_notes file
        # and delete them prematurely — misattributing them and stealing
        # them from the next release. github.sha is the commit that
        # triggered the workflow run (same commit create-release rendered
        # against), so the set of fragments is identical.
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ github.sha }}
          fetch-depth: 1
          # Drop the credential helper that `actions/checkout` configures by
          # default. Without this, the job's GITHUB_TOKEN is persisted into
          # `.git/config` for the duration of the run; if any later step in
          # this job uploads the workspace as an artifact (or anything else
          # later reads `.git/config`), the token leaks. Closes zizmor
          # finding `zizmor/artipacked` (alert #4655). The downstream
          # `peter-evans/create-pull-request` step accepts an explicit
          # `token:` input and does not rely on the persisted git helper,
          # so removing this credential has no functional impact.
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.12'

      - name: Install towncrier
        run: pip install towncrier==24.8.0

      - name: Render and consume fragments
        env:
          RELEASE_VERSION: ${{ needs.build.outputs.version }}
        run: |
          set -euo pipefail
          if [[ ! "$RELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+][a-zA-Z0-9.+_-]+)?$ ]]; then
            echo "ERROR: RELEASE_VERSION '$RELEASE_VERSION' is not a valid bare semver string" >&2
            exit 1
          fi
          # Skip cleanly when there are no fragments — nothing to clean up,
          # no PR needed. README.md is preserved per the contributor guide
          # in changelog.d/README.md.
          if [[ "$(find changelog.d -maxdepth 1 -name '*.md' ! -name 'README.md' | wc -l)" -eq 0 ]]; then
            echo "No changelog fragments to clean up — skipping PR"
            echo "skip=true" >> "$GITHUB_ENV"
            exit 0
          fi
          towncrier build --yes --version "$RELEASE_VERSION"
          echo "skip=false" >> "$GITHUB_ENV"

      - name: Open cleanup PR
        if: env.skip != 'true'
        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          # Pin base explicitly: when this workflow is triggered by a tag
          # push, github.sha may not sit on main HEAD, and the action's
          # default-branch resolution could pick a non-main base. We always
          # want this PR to target main.
          base: main
          commit-message: "chore: clear changelog fragments for ${{ needs.build.outputs.version }}"
          branch: "chore/post-release-cleanup-${{ needs.build.outputs.version }}"
          delete-branch: true
          title: "chore: clear changelog fragments for ${{ needs.build.outputs.version }}"
          body: |
            ## Summary
            Post-release housekeeping for **${{ needs.build.outputs.version }}**:

            - Renders `changelog.d/` fragments into `docs/release_notes/${{ needs.build.outputs.version }}.md`
            - Removes the consumed fragments

            The same render was already used to compose the GitHub release body for
            [${{ needs.build.outputs.tag }}](https://github.com/${{ github.repository }}/releases/tag/${{ needs.build.outputs.tag }}).
            This PR exists only to persist the cleanup so the next release does not
            re-render the same fragments.

            Auto-merge-friendly: no review value beyond a sanity check that the
            rendered notes look right.
          labels: |
            automation
            maintenance

  # NOTE: trigger-prerelease-docker and the old combined trigger-workflows
  # (which used to dispatch BOTH publish-docker AND publish-pypi) were
  # removed in the build-once-promote + atomicity refactors.
  # prerelease-docker.yml and docker-publish.yml are now invoked as
  # reusable workflows earlier in this file. Only PyPI publishing remains
  # on repository_dispatch — PyPI Trusted Publishing requires the publish
  # step to run in a top-level (non-reusable) workflow.

  # Dispatch PyPI publish via repository_dispatch. Kept as a dispatch
  # (rather than a reusable workflow_call) because PyPI's Trusted
  # Publisher matches the OIDC `workflow_ref` claim, which points to the
  # CALLER when a workflow is invoked via workflow_call — so a reusable
  # publish.yml would fail with `invalid-publisher`. Tracked in
  # pypa/gh-action-pypi-publish#166 and pypi/warehouse#11096.
  trigger-pypi:
    needs: [build, prerelease-docker, publish-docker]
    if: ${{ !cancelled() && needs.prerelease-docker.result == 'success' && needs.publish-docker.result == 'success' && needs.build.outputs.release_exists == 'false' }}
    runs-on: ubuntu-latest
    environment: release
    permissions:
      contents: read
    outputs:
      dispatch_time: ${{ steps.dispatch-time.outputs.timestamp }}

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      # Recorded BEFORE the dispatch so monitor-pypi can filter on createdAt
      # without missing the run created during this job.
      - name: Record dispatch time
        id: dispatch-time
        run: echo "timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"

      - name: Trigger PyPI publish workflow
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          RELEASE_TAG: ${{ needs.build.outputs.tag }}
          RELEASE_SHA: ${{ github.sha }}
        with:
          # PAT_TOKEN: repository_dispatch from a workflow run cannot be
          # fired with GITHUB_TOKEN — the API rejects it to prevent
          # workflow-trigger loops. Minimum scope: `repo` (full scope —
          # `public_repo` is rejected by createDispatchEvent regardless
          # of repo visibility).
          github-token: ${{ secrets.PAT_TOKEN }}
          script: |
            // Forward `prerelease: false` explicitly. publish.yml gates
            // Test PyPI vs prod PyPI on `client_payload.prerelease == true`;
            // if absent, the expression evaluates to '' and falls through to
            // prod PyPI. Setting false here makes the choice explicit. The
            // current pipeline only releases stable versions through this
            // dispatch — true prereleases (if added later) would need a
            // separate trigger path that flips this flag.
            //
            // `sha` pins publish.yml's checkout to the release commit.
            // repository_dispatch runs on default-branch HEAD *at dispatch
            // time*, and the `release` env approval is human-paced — on
            // v1.9.0 the approval came ~30h after the release commit, so
            // publish.yml built from a HEAD that was 4 PRs ahead of the
            // tag: PyPI 1.9.0 contains code that is not in the v1.9.0
            // tag, Docker image, or release notes. The SHA (not the tag)
            // must be passed because create-release creates the git tag
            // only AFTER PyPI succeeds — the tag does not exist yet here.
            await github.rest.repos.createDispatchEvent({
              owner: context.repo.owner,
              repo: context.repo.repo,
              event_type: 'publish-pypi',
              client_payload: {
                tag: process.env.RELEASE_TAG,
                sha: process.env.RELEASE_SHA,
                prerelease: false
              }
            });
            console.log('Triggered PyPI publish workflow');

  # Block on the dispatched publish.yml run so create-release downstream
  # only fires once PyPI has actually shipped. If PyPI fails, this job
  # fails and create-release is skipped — preventing the GH Release from
  # publishing with a missing PyPI artifact. (Docker promote already
  # blocked synchronously above via publish-docker reusable call.)
  monitor-pypi:
    needs: [build, trigger-pypi]
    if: ${{ !cancelled() && needs.trigger-pypi.result == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 90
    permissions:
      contents: read
      actions: read   # Required for gh run list
      issues: write

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Wait for PyPI publish workflow to complete
        id: wait
        env:
          GH_TOKEN: ${{ github.token }}
          # gh CLI cannot infer the repo here (this job has no checkout step),
          # and it does NOT fall back to GITHUB_REPOSITORY. Without GH_REPO,
          # `gh run list` fails with "failed to determine base repo", the
          # error is swallowed, and every poll sees an empty result.
          GH_REPO: ${{ github.repository }}
          RELEASE_TAG: ${{ needs.build.outputs.tag }}
          DISPATCH_TIME: ${{ needs.trigger-pypi.outputs.dispatch_time }}
        run: |
          # pipefail is essential here: the poll loop pipes `gh run list`
          # into `jq`, and without pipefail a transient `gh` failure
          # (network, auth, rate limit) is swallowed by `jq` returning
          # empty input, causing the loop to spin silently for the full
          # 40-minute budget rather than surfacing the error immediately.
          set -euo pipefail
          echo "Monitoring publish.yml for tag $RELEASE_TAG (dispatched at $DISPATCH_TIME)..."

          # Wait for the dispatched run to start (repository_dispatch is async)
          sleep 30

          max_wait=2400  # 40 minutes
          elapsed=0
          conclusion=""

          while [ "$elapsed" -lt "$max_wait" ]; do
            # Compare timestamps numerically — lexicographic comparison
            # breaks when GitHub returns sub-second precision (e.g.
            # "...:56.500Z" sorts before "...:56Z"). fromdateiso8601
            # doesn't accept fractional seconds, so strip them first. The
            # 60s buffer tolerates minor clock skew between the runner and
            # GitHub's API. Stderr is intentionally NOT redirected so gh
            # failures stay visible in the runner log.
            run=$(gh run list --workflow=publish.yml --limit=20 --json status,conclusion,createdAt \
              | jq -r --arg since "$DISPATCH_TIME" '
                  def to_epoch: sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601;
                  (($since | to_epoch) - 60) as $s
                  | [.[] | select((.createdAt | to_epoch) >= $s)] | .[0]
                ')

            if [ "$run" = "null" ] || [ -z "$run" ]; then
              echo "publish.yml: waiting for run to appear..." >&2
              sleep 30
              elapsed=$((elapsed + 30))
              continue
            fi

            status=$(echo "$run" | jq -r '.status')
            conclusion=$(echo "$run" | jq -r '.conclusion')

            if [ "$status" = "completed" ]; then
              echo "publish.yml: $conclusion" >&2
              break
            fi

            sleep 30
            elapsed=$((elapsed + 30))
          done

          if [ -z "$conclusion" ]; then
            echo "::error::publish.yml run did not start or complete within ${max_wait}s"
            echo "pypi_result=timed_out" >> "$GITHUB_OUTPUT"
            exit 1
          fi

          echo "pypi_result=$conclusion" >> "$GITHUB_OUTPUT"
          if [ "$conclusion" = "success" ]; then
            echo "PyPI publish completed successfully"
            exit 0
          else
            echo "::error::PyPI publish completed with conclusion: $conclusion"
            exit 1
          fi

      - name: Create issue on PyPI publish failure
        # Always run on failure so maintainers see a clear, persistent
        # record that captures the workflow run ID. The Actions UI shows
        # the failure too but issues are easier to search and triage.
        if: failure()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          RELEASE_TAG: ${{ needs.build.outputs.tag }}
          PYPI_RESULT: ${{ steps.wait.outputs.pypi_result }}
        with:
          script: |
            const { owner, repo } = context.repo;
            const tag = process.env.RELEASE_TAG;
            const pypi = process.env.PYPI_RESULT || 'unknown';
            const title = `PyPI publish failure for ${tag}`;

            // publish.yml fails closed without a full 40-hex client_payload.sha
            // (the pinned release commit — NOT main HEAD, which may have moved
            // during the approval wait). Embed the exact ready-to-paste
            // re-dispatch command so an on-call maintainer under incident
            // pressure cannot follow a stale two-field payload shape.
            const redispatch = '```\ngh api repos/' + owner + '/' + repo + '/dispatches \\\n'
              + "  -f event_type=publish-pypi \\\n"
              + "  -F 'client_payload[tag]=" + tag + "' \\\n"
              + "  -F 'client_payload[sha]=" + context.sha + "'\n```";
            const action = pypi === 'timed_out'
              ? '**Suggested action:** publish.yml did not complete within 40 minutes. Check the [Actions tab](https://github.com/' + owner + '/' + repo + '/actions)'
                + ' — if the run never appeared, this likely indicates a dispatch issue; re-trigger with:\n' + redispatch
                + '\nIf still running, no action may be needed.'
              : '**Suggested action:** Inspect publish.yml logs in the [Actions tab](https://github.com/' + owner + '/' + repo + '/actions), fix the underlying cause,'
                + ' then either (a) re-dispatch publish.yml with:\n' + redispatch
                + '\nand re-run release.yml to complete create-release, or (b) manually publish the GitHub Release if Docker promote also succeeded.'
                + ' See docs/RELEASE_GUIDE.md "Recovery from PyPI failure".';

            const body = `## PyPI publish failed for ${tag}\n\nResult: \`${pypi}\`\n\nNote: At this point in the atomicity flow, Docker promote (publish-docker job) has already succeeded — the Docker release tags exist and are signed. Only PyPI and the GitHub Release are missing.\n\n${action}`;

            // De-dup: if an open issue with the same title already exists,
            // comment on it instead of opening a duplicate.
            const existing = await github.paginate(github.rest.issues.listForRepo, {
              owner, repo, state: 'open', labels: 'ci-cd', per_page: 100,
            });
            const match = existing.find(i => i.title === title && !i.pull_request);

            if (match) {
              await github.rest.issues.createComment({
                owner, repo, issue_number: match.number,
                body: `Re-detected on workflow run [${context.runId}](https://github.com/${owner}/${repo}/actions/runs/${context.runId}):\n\n${body}`,
              });
            } else {
              await github.rest.issues.create({
                owner, repo, title, body, labels: ['ci-cd'],
              });
            }

  # ============================================================================
  # CLEANUP ON REJECTION
  # ============================================================================
  # In the build-once-promote model, prerelease-docker signs the manifest
  # BEFORE the publish step runs. If publish-docker (the retag step) fails
  # or the maintainer rejects the `release` env approval, the prerelease
  # tag and its cosign artifacts (`sha256-<digest>.{sig,att}`) are left
  # orphaned on Docker Hub forever — the existing cleanup loop inside
  # docker-publish.yml only runs on its success path.
  #
  # SAFETY: cosign signature/attestation artifacts are stored at a tag named
  # `sha256-<manifest-digest>.{sig,att}` and discovered by manifest digest,
  # NOT by image tag. After publish-docker SUCCEEDS, the release tags
  # (`:VERSION`, `:MAJOR_MINOR`, `:latest`) share the prerelease manifest
  # digest (imagetools retag preserves the digest), so the cosign artifacts
  # anchor BOTH the deleted prerelease tag AND the live release tags.
  # Deleting them after a successful retag would invalidate release-tag
  # cosign verification.
  #
  # Therefore: this job ONLY fires when publish-docker did not succeed.
  # Beyond that point, docker-publish.yml's success-path cleanup handles
  # prerelease-tag deletion, and cosign artifacts must stay.
  #
  # Edge case: partial retag failure (e.g., `:1.6.9` lands but `:latest`
  # fails) — publish-docker exits failure with some release tags already
  # created. The cleanup script enumerates the three possible release tags
  # against Docker Hub and rolls back any that exist BEFORE deleting
  # cosign artifacts. This is the only case where we delete release tags
  # rather than leave them.
  # ============================================================================
  cleanup-on-rejection:
    name: Clean up orphan prerelease tags and signatures
    needs: [build, prerelease-docker, publish-docker]
    if: >-
      ${{ always()
          && needs.prerelease-docker.result != 'skipped'
          && needs.build.outputs.release_exists == 'false'
          && (needs.prerelease-docker.result == 'failure'
              || needs.prerelease-docker.result == 'cancelled'
              || needs.publish-docker.result == 'failure'
              || needs.publish-docker.result == 'cancelled') }}
    runs-on: ubuntu-latest
    # DOCKER_USERNAME / DOCKER_PASSWORD are env-scoped to `release`
    # (deliberately not repo-level — see comment on the build job above).
    # Without `environment: release` here, those secrets resolve to empty
    # strings and the Docker Hub login at the bottom of this job exits 1,
    # leaving the orphan tags + cosign artifacts the cleanup is meant to
    # remove. The `release` env approval was already granted upstream in
    # this run, so this does not add a new prompt.
    environment: release
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Roll back partial release tags then delete prerelease + cosign artifacts
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
          VERSION: ${{ needs.build.outputs.version }}
          SHORT_SHA: ${{ needs.build.outputs.short_sha }}
          DIGEST: ${{ needs.prerelease-docker.outputs.manifest_digest }}
        run: |
          set -euo pipefail
          # Authenticate with Docker Hub API (same JWT flow as docker-publish.yml cleanup)
          TOKEN=$(curl -sS -X POST -H "Content-Type: application/json" \
            -d "{\"username\":\"${DOCKER_USERNAME}\",\"password\":\"${DOCKER_PASSWORD}\"}" \
            https://hub.docker.com/v2/users/login/ | jq -r .token)

          if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
            echo "::error::Failed to authenticate with Docker Hub API — cannot clean up orphans"
            exit 1
          fi

          REPO="${DOCKER_USERNAME}/local-deep-research"
          MANIFEST_TAG="prerelease-v${VERSION}-${SHORT_SHA}"
          MAJOR_MINOR=$(echo "$VERSION" | cut -d. -f1,2)

          delete_tag() {
            local TAG="$1"
            local STATUS
            STATUS=$(curl -sS -o /dev/null -w "%{http_code}" -X DELETE \
              -H "Authorization: JWT ${TOKEN}" \
              "https://hub.docker.com/v2/repositories/${REPO}/tags/${TAG}/" || echo "ERR")
            case "$STATUS" in
              200|204) echo "Deleted ${TAG}";;
              404)     echo "Skip ${TAG} (already absent — expected)";;
              401|403)
                echo "::error::Auth failure deleting ${TAG} — DOCKER_PASSWORD may be missing Delete scope on the Docker Hub PAT"
                exit 1
                ;;
              *)       echo "::warning::Unexpected HTTP ${STATUS} for ${TAG}";;
            esac
          }

          # =====================================================================
          # STEP 1 — Partial retag rollback.
          # =====================================================================
          # If publish-docker failed mid-way through `docker buildx imagetools
          # create -t :VERSION -t :MAJOR_MINOR -t :latest`, one or two of the
          # three release tags may have landed. Those tags would point at the
          # prerelease manifest digest, which we're about to delete cosign
          # artifacts for. We MUST roll back any landed release tags BEFORE
          # touching cosign — otherwise the rollback leaves them broken.
          #
          # If publish-docker was skipped (prerelease failed before reaching
          # retag), the release tags can't exist, so this loop is a cheap no-op
          # (three 404 responses).
          # =====================================================================
          echo "STEP 1 — Roll back any release tags that landed during a partial retag..."
          for RELEASE_TAG in "${VERSION}" "${MAJOR_MINOR}" "latest"; do
            # HEAD-check before DELETE so the "expected absent" case is silent.
            # Docker Hub's tag endpoint returns 200 on HEAD if tag exists, 404 if not.
            CHECK=$(curl -sS -o /dev/null -w "%{http_code}" \
              -H "Authorization: JWT ${TOKEN}" \
              "https://hub.docker.com/v2/repositories/${REPO}/tags/${RELEASE_TAG}/" || echo "ERR")
            if [ "$CHECK" = "200" ]; then
              echo "::warning::Release tag ${RELEASE_TAG} exists from a partial retag — rolling back"
              delete_tag "${RELEASE_TAG}"
            else
              echo "Release tag ${RELEASE_TAG}: not present (HTTP ${CHECK}) — nothing to roll back"
            fi
          done

          # =====================================================================
          # STEP 2 — Prerelease tag cleanup.
          # =====================================================================
          # Always-safe targets: the prerelease manifest list + its per-arch
          # children + the floating `:prerelease` tag. Cleanup is best-effort
          # here (404s are normal — e.g., build failed before pushing arm64).
          #
          # The floating `:prerelease` tag (re-pointed by prerelease-docker.yml
          # to the current run's manifest) is included so a rejected release
          # does not leave `:prerelease` pointing at a manifest whose cosign
          # artifacts step 4 below is about to delete — pulling `:prerelease`
          # in that window would yield an image whose signature is gone, and
          # the README cosign-verify recipe would fail. Returning 404 on
          # `:prerelease` is unambiguously safer than serving an unverifiable
          # image; the next successful prerelease-docker run re-creates it.
          # =====================================================================
          echo "STEP 2 — Delete prerelease tags..."
          for TAG in "${MANIFEST_TAG}" "${MANIFEST_TAG}-amd64" "${MANIFEST_TAG}-arm64" "prerelease"; do
            delete_tag "${TAG}"
          done

          # =====================================================================
          # STEP 3 — Discover per-arch digests from the manifest list BEFORE
          # we delete cosign artifacts.
          # =====================================================================
          # prerelease-docker.yml's per-arch SBOM step attests against each
          # per-arch digest, producing artifacts at
          # `sha256-<per-arch-digest>.{sig,att,sbom}` in addition to the
          # manifest-list-digest artifacts. Need to enumerate from the manifest
          # while it still exists (we just deleted the TAG, but the manifest
          # body persists until Docker Hub GC, and we can inspect by digest).
          # Use the captured DIGEST to inspect the manifest list directly.
          # =====================================================================
          PER_ARCH_TAGS=()
          if [[ -n "${DIGEST:-}" && "$DIGEST" == sha256:* ]]; then
            echo "STEP 3 — Discovering per-arch digests from manifest list ${DIGEST}..."
            if docker buildx imagetools inspect "${REPO}@${DIGEST}" --raw > /tmp/manifest.json 2>/dev/null; then
              while IFS= read -r PER_ARCH_DIGEST; do
                if [[ -n "${PER_ARCH_DIGEST}" && "${PER_ARCH_DIGEST}" == sha256:* ]]; then
                  PER_ARCH_TAG_PREFIX="${PER_ARCH_DIGEST/:/-}"
                  PER_ARCH_TAGS+=(
                    "${PER_ARCH_TAG_PREFIX}.sig"
                    "${PER_ARCH_TAG_PREFIX}.att"
                    "${PER_ARCH_TAG_PREFIX}.sbom"
                  )
                  echo "  queued cleanup for per-arch artifacts at ${PER_ARCH_DIGEST}"
                fi
              done < <(jq -r '.manifests[] | select(.platform.architecture != "unknown") | .digest' /tmp/manifest.json)
            else
              echo "::warning::Could not inspect manifest at ${DIGEST} — skipping per-arch attestation cleanup"
            fi
          else
            echo "STEP 3 — No valid manifest digest captured (got '${DIGEST:-<empty>}'); skipping per-arch discovery"
          fi

          # =====================================================================
          # STEP 4 — Cosign signature/attestation artifact cleanup.
          # =====================================================================
          # Safe to delete only because STEP 1 already rolled back any release
          # tags that might be sharing these digests. The .sbom entries are
          # legacy from `cosign attach sbom` (current code uses `cosign attest
          # --type spdxjson` which writes to .att) — kept as belt-and-suspenders
          # cleanup for any leftovers from older releases.
          # =====================================================================
          COSIGN_TAGS=()
          if [[ -n "${DIGEST:-}" && "$DIGEST" == sha256:* ]]; then
            DIGEST_TAG_PREFIX="${DIGEST/:/-}"
            COSIGN_TAGS+=(
              "${DIGEST_TAG_PREFIX}.sig"
              "${DIGEST_TAG_PREFIX}.att"
              "${DIGEST_TAG_PREFIX}.sbom"
            )
          fi
          COSIGN_TAGS+=("${PER_ARCH_TAGS[@]}")

          if [ "${#COSIGN_TAGS[@]}" -gt 0 ]; then
            echo "STEP 4 — Delete cosign signature/attestation artifacts..."
            for TAG in "${COSIGN_TAGS[@]}"; do
              delete_tag "${TAG}"
            done
          else
            echo "STEP 4 — No cosign artifact tags to clean up (no captured digest)"
          fi
          # NOTE: NO `continue-on-error: true` at job level — auth failures
          # (401/403) are loud so missing token scopes can't silently let
          # orphans accumulate forever.