learningcircuit--local-deep-research/.github/workflows/compose-integration-test.yml

name: Compose Integration Test

# Brings up the bundled docker-compose.yml end-to-end in CI and verifies the
# whole stack (searxng, ollama, local-deep-research) reaches a healthy state.
# This is the test that would have caught #3874 (broken cap_drop on searxng)
# before users hit it. See #3886 for the rationale.
#
# Cost: ~3-6 min per run depending on cache state. Runs only via the release
# pipeline (`release.yml` includes it as `compose-integration-gate`) and on
# manual dispatch. NOT inside `release-gate.yml`, because release-gate runs
# daily and the failure modes here (compose/image changes) are tied to
# actual release events, not time. NOT on pull_request — too expensive for
# per-PR feedback latency.
#
# Do not move this into release-gate's daily cron. Ongoing compose drift
# between releases is already covered by `compose-published-smoke.yml`,
# which runs weekly against main's compose.yml + the *published* Docker Hub
# image — the more meaningful drift to catch, since users pull the
# published image, not main's build override. PR #3962 tried adding this
# to the daily cron and was closed for that reason.
#
# Tests the docker-compose.yml as users actually run it — no test-only
# overrides. The model pre-pull was removed from the compose itself in this
# same PR so the stack starts in seconds rather than waiting on a multi-GB
# download that the test doesn't need.

on:
  workflow_call:
  workflow_dispatch:

permissions:
  contents: read

jobs:
  compose-up:
    name: docker compose up + healthcheck
    runs-on: ubuntu-latest
    timeout-minutes: 25

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      # Build the LDR image from the working tree and tag it as the name the
      # bundled compose references. Compose then uses the local image instead
      # of pulling the published one — so we test the current code path, not
      # whatever's on Docker Hub.
      - name: Build LDR image with the published tag
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
        with:
          context: .
          # Same target as docker-tests.yml's `ldr-prod` build, so its cache
          # layers (scope=ldr-prod, populated by ci-gate during the same
          # release pipeline) are reusable here. Explicit target guards
          # against future Dockerfile reorderings making the default stage
          # something other than `ldr`.
          target: ldr
          load: true
          tags: localdeepresearch/local-deep-research:latest
          # Read from both our own scope and the shared `ldr-prod` scope —
          # whichever has the layers wins. Falls back to a fresh build if
          # neither does (e.g. on a brand-new branch). Only write to our
          # own scope so we don't poison the cross-workflow cache.
          cache-from: |
            type=gha,scope=compose-integration
            type=gha,scope=ldr-prod
          cache-to: type=gha,mode=max,scope=compose-integration

      - name: Bring up the stack
        # `docker compose up -d` pulls any image it doesn't have locally
        # (default pull_policy: missing), so ollama and searxng are pulled
        # inline. The LDR image is the locally-built one from the previous
        # step — compose sees it's already present and uses it as-is.
        # `--no-build` defends against a future docker-compose.yml change
        # adding a `build:` directive — we want the image we tagged above,
        # not a fresh build that bypasses the cache strategy.
        run: docker compose up -d --no-build

      - name: Wait for the stack to be healthy and serving
        # Budget: 6 min covers cold daemon startup + LDR migrations + flask
        # boot + slow CI runners. Without the model pull this completes in
        # ~1-2 min on a warm runner.
        #
        # Container resolution: always via `docker compose ps -q <service>`
        # so we don't couple to compose's `container_name:` values. If those
        # drift, this still works.
        run: |
          set -euo pipefail
          deadline=$(( $(date +%s) + 360 ))

          cid_for() { docker compose ps -q "$1" 2>/dev/null || true; }
          status() {
            local cid=$1
            [ -n "$cid" ] || { echo missing; return; }
            docker inspect -f "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" "$cid" 2>/dev/null || echo missing
          }

          while :; do
            now=$(date +%s)
            if [ "$now" -ge "$deadline" ]; then
              echo "::error::Timed out after 6 min waiting for stack to be healthy"
              docker compose ps
              exit 1
            fi

            ollama_id=$(cid_for ollama)
            searxng_id=$(cid_for searxng)
            ldr_id=$(cid_for local-deep-research)

            ollama_h=$(status "$ollama_id")
            searxng_h=$(status "$searxng_id")
            ldr_h=$(status "$ldr_id")

            echo "[$(date -u +%H:%M:%S)] ollama=$ollama_h searxng=$searxng_h ldr=$ldr_h"

            # All three services have healthchecks: ollama and searxng via
            # docker-compose.yml, LDR via the Dockerfile (HEALTHCHECK at
            # Dockerfile:306, probing /api/v1/health). status() returns the
            # health status when one is defined, so require "healthy" for
            # all three — strictly stronger signal than "running".
            if [ "$ollama_h" = "healthy" ] && [ "$searxng_h" = "healthy" ] && [ "$ldr_h" = "healthy" ]; then
              echo "All services healthy."
              break
            fi

            # Fail fast on any container exiting non-zero rather than burning
            # the whole 6 min budget.
            for svc in ollama searxng local-deep-research; do
              cid=$(cid_for "$svc")
              [ -n "$cid" ] || continue
              s=$(docker inspect -f '{{.State.Status}}' "$cid" 2>/dev/null || true)
              if [ "$s" = "exited" ] || [ "$s" = "dead" ]; then
                echo "::error::Container for service $svc has exited"
                exit 1
              fi
            done

            sleep 10
          done

      - name: Probe LDR HTTP endpoint
        # Avoids `curl ... | grep` so we don't need pipefail to surface curl
        # failures — the HTTP code is captured directly via -w and checked
        # with a case statement. `|| echo "000"` is the sentinel for true
        # network failures (connection refused, DNS, etc.) so we can log
        # on retry.
        #
        # Deliberately no `-f`: with `-f`, curl exits non-zero on HTTP
        # 4xx/5xx AND suppresses -w output — which would collapse "404",
        # "503", and network-error all into "000" and erase the most
        # interesting failure signal (LDR up but serving an error page).
        # Without -f, every HTTP response gives us its real code; only
        # true network failures fall through to "000".
        run: |
          set -euo pipefail
          for i in $(seq 1 30); do
            code=$(curl -sS -o /dev/null -w "%{http_code}" http://localhost:5000/ 2>/dev/null || echo "000")
            case "$code" in
              200|301|302|303|307|308)
                echo "LDR is serving on :5000 (HTTP $code)"
                exit 0
                ;;
            esac
            echo "Waiting for LDR HTTP (attempt $i/30, last code: $code)..."
            sleep 5
          done
          echo "::error::LDR HTTP probe failed after ~150s"
          exit 1

      - name: Dump compose state and logs
        if: always()
        run: |
          echo "::group::docker compose ps"
          docker compose ps || true
          echo "::endgroup::"

          for svc in ollama searxng local-deep-research; do
            echo "::group::$svc logs"
            docker compose logs --no-color --tail=500 "$svc" || true
            echo "::endgroup::"
          done

      - name: Tear down
        if: always()
        run: docker compose down -v --remove-orphans