learningcircuit--local-deep-research/.github/workflows/compose-published-smoke.yml

name: Compose Published-Image Smoke

# Tests "main's docker-compose.yml + the currently-published Docker Hub image"
# end-to-end. Catches drift between compose changes that have landed on main
# and the image artefact that users actually pull when they follow the README
# quickstart:
#
#   curl -O .../docker-compose.yml && docker compose up -d
#
# This complements the release-pipeline test in compose-integration-test.yml,
# which builds the image from the working tree (validating "this code's
# compose with this code's image") and runs only at release time + manual
# dispatch. That test cannot catch the drift case — the published image
# always lags main between releases.
#
# Triggers: weekly schedule + manual dispatch only. No pull_request trigger
# because PRs don't change the published image — by definition this test
# can only fail on drift that's already landed on main.
#
# On failure, opens a tracking issue (or comments on the existing one) so a
# scheduled-job failure isn't lost in the noise.

on:
  workflow_dispatch:
  schedule:
    # 05:00 UTC every Monday — offset from release-gate's 02:00 daily cron
    # to avoid runner contention. Weekly cadence is enough since the failure
    # modes are slow-moving (compose-vs-published-image drift) and the test
    # itself burns CI minutes.
    - cron: '0 5 * * 1'

permissions:
  contents: read

jobs:
  compose-up:
    name: docker compose up + healthcheck (published image)
    runs-on: ubuntu-latest
    timeout-minutes: 25
    permissions:
      contents: read
      issues: write  # for the failure-issue step

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout (for docker-compose.yml on default branch)
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      # Pull all three services from their registries, including the LDR
      # image. This is the whole point of this workflow — exercise the
      # exact artefact users get from `docker compose up -d`.
      - name: Pull all images
        run: docker compose pull

      - name: Bring up the stack
        # `--no-build` defends against a future docker-compose.yml change
        # adding a `build:` directive — this workflow specifically tests the
        # *published* image artifact, not a fresh build from source.
        run: docker compose up -d --no-build

      - name: Wait for the stack to be healthy and serving
        # Same wait logic as compose-integration-test.yml. 6 min budget
        # covers cold daemon startup + LDR migrations + flask boot.
        # Container resolution: always via `docker compose ps -q <service>`
        # so we don't couple to compose's `container_name:` values.
        run: |
          set -euo pipefail
          deadline=$(( $(date +%s) + 360 ))

          cid_for() { docker compose ps -q "$1" 2>/dev/null || true; }
          status() {
            local cid=$1
            [ -n "$cid" ] || { echo missing; return; }
            docker inspect -f "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" "$cid" 2>/dev/null || echo missing
          }

          while :; do
            now=$(date +%s)
            if [ "$now" -ge "$deadline" ]; then
              echo "::error::Timed out after 6 min waiting for stack to be healthy"
              docker compose ps
              exit 1
            fi

            ollama_id=$(cid_for ollama)
            searxng_id=$(cid_for searxng)
            ldr_id=$(cid_for local-deep-research)

            ollama_h=$(status "$ollama_id")
            searxng_h=$(status "$searxng_id")
            ldr_h=$(status "$ldr_id")

            echo "[$(date -u +%H:%M:%S)] ollama=$ollama_h searxng=$searxng_h ldr=$ldr_h"

            # All three services have healthchecks: ollama and searxng via
            # docker-compose.yml, LDR via the Dockerfile (HEALTHCHECK at
            # Dockerfile:306, probing /api/v1/health). status() returns the
            # health status when one is defined, so require "healthy" for
            # all three.
            if [ "$ollama_h" = "healthy" ] && [ "$searxng_h" = "healthy" ] && [ "$ldr_h" = "healthy" ]; then
              echo "All services healthy."
              break
            fi

            for svc in ollama searxng local-deep-research; do
              cid=$(cid_for "$svc")
              [ -n "$cid" ] || continue
              s=$(docker inspect -f '{{.State.Status}}' "$cid" 2>/dev/null || true)
              if [ "$s" = "exited" ] || [ "$s" = "dead" ]; then
                echo "::error::Container for service $svc has exited"
                exit 1
              fi
            done

            sleep 10
          done

      - name: Probe LDR HTTP endpoint
        # Avoids `curl ... | grep` so we don't need pipefail to surface curl
        # failures — the HTTP code is captured directly via -w and checked
        # with a case statement. `|| echo "000"` is the sentinel for true
        # network failures (connection refused, DNS, etc.) so we can log
        # on retry.
        #
        # Deliberately no `-f`: with `-f`, curl exits non-zero on HTTP
        # 4xx/5xx AND suppresses -w output — which would collapse "404",
        # "503", and network-error all into "000" and erase the most
        # interesting failure signal (LDR up but serving an error page).
        # Without -f, every HTTP response gives us its real code; only
        # true network failures fall through to "000".
        run: |
          set -euo pipefail
          for i in $(seq 1 30); do
            code=$(curl -sS -o /dev/null -w "%{http_code}" http://localhost:5000/ 2>/dev/null || echo "000")
            case "$code" in
              200|301|302|303|307|308)
                echo "LDR is serving on :5000 (HTTP $code)"
                exit 0
                ;;
            esac
            echo "Waiting for LDR HTTP (attempt $i/30, last code: $code)..."
            sleep 5
          done
          echo "::error::LDR HTTP probe failed after ~150s"
          exit 1

      - name: Dump compose state and logs
        if: always()
        run: |
          echo "::group::docker compose ps"
          docker compose ps || true
          echo "::endgroup::"

          for svc in ollama searxng local-deep-research; do
            echo "::group::$svc logs"
            docker compose logs --no-color --tail=500 "$svc" || true
            echo "::endgroup::"
          done

          # Capture image digests for both the workflow log (every run, via
          # the cat below) and the auto-failure-issue body (failure runs
          # only, via the next step). Always-logging gives us audit and
          # bisection signal — "what was the published image SHA when this
          # passed on date X?" — without needing to re-run the workflow.
          # Resolve every container by service name through `docker compose
          # ps -q` so we don't depend on `container_name:` values.
          cid_for() { docker compose ps -q "$1" 2>/dev/null || true; }
          {
            echo "## Image digests"
            for svc in ollama searxng local-deep-research; do
              cid=$(cid_for "$svc")
              if [ -n "$cid" ]; then
                img=$(docker inspect -f '{{.Image}}' "$cid" 2>/dev/null || echo missing)
                echo "- $svc: image=$img container=$cid"
              else
                echo "- $svc: container=missing"
              fi
            done
          } > /tmp/digests.md
          echo "::group::Image digests"
          cat /tmp/digests.md
          echo "::endgroup::"

      - name: Open or update failure-tracking issue
        if: failure() && github.event_name == 'schedule'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: ${{ github.repository }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -euo pipefail
          # Stable title prefix so we dedup across weeks. If an issue with
          # this title is already open, comment on it; otherwise create a
          # fresh one. We don't auto-close on success — let a human triage.
          TITLE="[compose-published-smoke] published-image drift detected"

          DATE=$(date -u +%Y-%m-%d)
          DIGESTS=$(cat /tmp/digests.md 2>/dev/null || echo "(digests not captured)")

          BODY=$(cat <<MARKDOWN
          The weekly **published-image smoke test** failed on $DATE.

          This usually means \`main\`'s \`docker-compose.yml\` and the currently-published
          \`localdeepresearch/local-deep-research:latest\` on Docker Hub have drifted —
          either a compose change landed without an image republish, or the
          published image (or one of its dependencies) regressed.

          **Workflow run:** $RUN_URL

          **What to check first:**
          - Compose changes since the last release: \`git log --oneline <last-release-tag>..main -- docker-compose.yml\`
          - Whether the LDR image needs a republish (cut a release, or the image was repointed)
          - Upstream image regressions: ollama, searxng image digests below

          $DIGESTS

          *This issue is auto-managed by \`.github/workflows/compose-published-smoke.yml\`.
          Subsequent failures will add comments here. Close it when the underlying drift is fixed
          and a successful run rolls in.*
          MARKDOWN
          )

          # Find existing open issue with the same title
          NUM=$(gh issue list -R "$REPO" \
            --state open \
            --search "in:title \"[compose-published-smoke] published-image drift detected\"" \
            --json number,title \
            --jq '.[0].number // empty' \
            --limit 1)

          if [ -n "$NUM" ]; then
            echo "Commenting on existing issue #$NUM"
            # `--body=` (= form) prevents gh from interpreting body content
            # as flags if it ever starts with `-`. Same below.
            gh issue comment "$NUM" -R "$REPO" --body="$BODY"
          else
            echo "Creating new issue"
            gh issue create -R "$REPO" \
              --title="$TITLE" \
              --label="bug,docker,ci-cd" \
              --body="$BODY"
          fi

      - name: Tear down
        # `|| true` so a teardown flake (daemon hiccup, hung container) can't
        # flip the job to failure() after the smoke test itself passed —
        # which would falsely trigger the auto-issue step. CI runners are
        # ephemeral, so any leftover containers/volumes vanish with the
        # runner regardless.
        if: always()
        run: docker compose down -v --remove-orphans || true