learningcircuit--local-deep-research/.github/workflows/docker-tests.yml

name: Docker Tests (Consolidated)

on:
  pull_request:
    types: [opened, synchronize, reopened]
    branches: [ main ]
  push:
    branches: [ main ]
  workflow_call:  # Called by ci-gate.yml for release pipeline
    inputs:
      strict-mode:
        description: 'Run in strict mode (all tests blocking, no continue-on-error)'
        required: false
        type: boolean
        default: true
    secrets:
      OPENROUTER_API_KEY:
        required: false
      GIST_TOKEN:
        required: false
      COVERAGE_GIST_ID:
        required: false
  workflow_dispatch:

# No concurrency group — intentionally omitted.
# This workflow triggers on both pull_request and workflow_call (from
# ci-gate.yml / release-gate.yml). A shared concurrency key would cause
# direct PR runs and workflow_call runs to cancel each other mid-flight.
# See #3554 (reverted in #3599) for context.

# Top-level permissions set to minimum (OSSF Scorecard Token-Permissions)
permissions: {}

env:
  TEST_IMAGE: ldr-test
  PAGES_URL: https://learningcircuit.github.io/local-deep-research/

jobs:
  # Detect which paths changed to conditionally run certain jobs
  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      llm: ${{ steps.filter.outputs.llm }}
      infrastructure: ${{ steps.filter.outputs.infrastructure }}
      docker: ${{ steps.filter.outputs.docker }}
      accessibility: ${{ steps.filter.outputs.accessibility }}
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        id: filter
        with:
          filters: |
            llm:
              - 'src/local_deep_research/llm/**'
              - 'src/local_deep_research/config/llm_config.py'
              - 'src/local_deep_research/api/research_functions.py'
              - 'tests/test_llm/**'
              - 'examples/**'
              - '.github/workflows/docker-tests.yml'
            infrastructure:
              - 'src/local_deep_research/web/routes/**'
              - 'src/local_deep_research/web/static/js/**'
              - 'tests/infrastructure_tests/**'
              - 'tests/js/**'
              - '.github/workflows/docker-tests.yml'
            docker:
              - 'Dockerfile'
              - '.dockerignore'
              - 'scripts/ldr_entrypoint.sh'
              - 'scripts/ollama_entrypoint.sh'
              - 'pyproject.toml'
              - 'pdm.lock'
              - 'package.json'
              - 'package-lock.json'
              - 'vite.config.js'
              - '.github/workflows/docker-tests.yml'
            accessibility:
              - 'src/local_deep_research/web/templates/**'
              - 'src/local_deep_research/web/static/css/**'
              - 'src/local_deep_research/web/static/js/**'
              - 'tests/accessibility_tests/**'
              - 'playwright.config.js'
              - 'lighthouserc.json'
              - '.github/workflows/docker-tests.yml'

  # Build Docker image once and share via artifact
  build-test-image:
    runs-on: ubuntu-latest
    name: Build Test Image
    if: |
      github.event_name == 'pull_request' ||
      github.ref == 'refs/heads/main' ||
      inputs.strict-mode == true

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: Build Docker image with cache
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
        with:
          context: .
          target: ldr-test
          outputs: type=docker,dest=/tmp/ldr-test.tar
          tags: ${{ env.TEST_IMAGE }}
          # Cache poisoning protection: only read/write cache on trusted events (not PRs from forks)
          cache-from: ${{ github.event_name != 'pull_request' && 'type=gha,scope=ldr-test' || '' }}
          cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=ldr-test' || '' }}

      - name: Upload Docker image as artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: docker-image
          path: /tmp/ldr-test.tar
          retention-days: 1

  # Run all pytest tests with coverage
  pytest-tests:
    runs-on: ubuntu-latest
    name: All Pytest Tests + Coverage
    # Cap exists so a hung run doesn't inherit GitHub's 360-minute default
    # (observed 2026-06-10 on PR #4491: a flaked run sat in the main pytest
    # step for 2+ hours before being cancelled manually). The original
    # 75-min cap (#4514) was sized from a 19-57 min historical range, but
    # healthy runs legitimately reach ~95 min (e.g. PR #4465 on 2026-06-07:
    # 1h35m in the pytest step alone; #4471/#4517 were killed mid-suite at
    # 75 min the day the cap merged). 120 min = worst observed healthy run
    # + ~25% headroom, while still ending a truly hung run 4x sooner than
    # the default.
    timeout-minutes: 120
    needs: build-test-image
    environment: ci
    permissions:
      contents: write      # Needed for gh-pages deployment
      pull-requests: write # Needed for PR comments

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: true  # Needed for gh-pages push

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      - name: Set up Node.js
        # zizmor: ignore[cache-poisoning] - caching explicitly disabled with cache: ''
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: '24'
          # Cache poisoning protection: explicitly disable caching on PR events from forks
          cache: ''

      - name: Install infrastructure test dependencies
        run: |
          cd tests/infrastructure_tests && npm ci

      - name: Run all pytest tests with coverage
        # Tests are always blocking - failures should not be silently ignored.
        # test_settings_routes_coverage.py is excluded here and run serially
        # in a dedicated step below — its 94 authenticated_client tests
        # (each one: register + login + SQLCipher KDF + 10 Alembic migrations)
        # accumulate enough connection/FD pressure inside the assigned xdist
        # worker to deadlock it silently late in the run (the worker dies
        # without a timeout, its coverage data is lost, total coverage drops
        # under the 50% fail-under gate, the job fails). Same pattern as
        # fd_canary tests, same fix: serial, no '-n auto' for this file.
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
        run: |
          mkdir -p coverage
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            -e GITHUB_ACTIONS=true \
            -e LDR_TESTING_WITH_MOCKS=true \
            -e LDR_DISABLE_RATE_LIMITING=true \
            -e PYTHONPATH=/app/src \
            -e OPENROUTER_API_KEY="${OPENROUTER_API_KEY}" \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/ \
              --cov=src \
              --cov-report= \
              --cov-fail-under=0 \
              --tb=short \
              --no-header \
              -q \
              --durations=20 \
              -n auto \
              -m 'not integration and not fd_canary' \
              --ignore=tests/ui_tests \
              --ignore=tests/infrastructure_tests \
              --ignore=tests/api_tests_with_login \
              --ignore=tests/fuzz \
              --ignore=tests/web/routes/test_settings_routes_coverage.py"

      # See the comment on the main pytest step above for why this file is
      # split out. Runs SERIAL (no '-n auto'), appends coverage data into
      # the main run's .coverage file, and emits the final xml + html
      # reports + the 50% fail-under check after both runs have contributed.
      - name: Run test_settings_routes_coverage.py (serial, with coverage)
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            -e GITHUB_ACTIONS=true \
            -e LDR_TESTING_WITH_MOCKS=true \
            -e LDR_DISABLE_RATE_LIMITING=true \
            -e PYTHONPATH=/app/src \
            -e OPENROUTER_API_KEY="${OPENROUTER_API_KEY}" \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/web/routes/test_settings_routes_coverage.py \
              --cov=src \
              --cov-append \
              --cov-report=xml:coverage/coverage.xml \
              --cov-report=html:coverage/htmlcov \
              --cov-fail-under=50 \
              --tb=short \
              --no-header \
              -q \
              --durations=20"

      # fd_canary tests (#3816 FD-leak guards) are CPU-bound real-event-loop
      # spins. Under coverage tracing + '-n auto' contention they blow past
      # the global 60s timeout and pressure xdist workers into crashing.
      # Run them in a dedicated SERIAL step with NO coverage and NO '-n auto'
      # so the FD-count signal stays deterministic (~10s) without flaking.
      - name: Run fd_canary FD-leak tests (serial, no coverage)
        if: always()
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            -e GITHUB_ACTIONS=true \
            -e LDR_TESTING_WITH_MOCKS=true \
            -e LDR_DISABLE_RATE_LIMITING=true \
            -e PYTHONPATH=/app/src \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/utilities/test_close_base_llm.py \
              -m fd_canary \
              -p no:cov \
              --tb=short \
              --no-header \
              -q"

      - name: Run JavaScript infrastructure tests
        if: always()
        run: |
          cd tests/infrastructure_tests && npm test

      - name: Run Vitest JS unit tests
        if: always()
        run: |
          npm ci && npm test

      - name: Generate coverage summary
        if: always()
        run: |
          {
            echo "## Coverage Report"
            echo ""
            if [ -f coverage/coverage.xml ]; then
              # Extract total coverage from XML
              COVERAGE=$(python3 -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage/coverage.xml'); print(f\"{float(tree.getroot().get('line-rate', 0)) * 100:.1f}%\")" 2>/dev/null || echo "N/A")
              echo "**Total Line Coverage: ${COVERAGE}**"
              echo ""
              echo "📊 **[View Full Coverage Report](${{ env.PAGES_URL }})** - Interactive HTML report with line-by-line details"
              echo ""
              echo "_Note: Report updates after merge to main/dev branch._"
            else
              echo "⚠️ Coverage report not generated"
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload coverage reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-report
          path: coverage/

      # Fix permissions on coverage directory (Docker creates files as root)
      - name: Fix coverage directory permissions
        if: always()
        run: |
          if [ -d "coverage" ]; then
            sudo chown -R "$(id -u):$(id -g)" coverage/
            echo "Coverage directory owner after chown:"
            stat coverage/
            if [ -d "coverage/htmlcov" ]; then
              echo "HTML coverage file count:"
              find coverage/htmlcov/ -type f | wc -l
              echo "Sample files:"
              find coverage/htmlcov/ -maxdepth 1 -type f -name "*.html" | head -5
            else
              echo "ERROR: coverage/htmlcov directory does not exist!"
            fi
          else
            echo "ERROR: coverage directory does not exist!"
          fi

      # Deploy coverage HTML report to GitHub Pages
      # Only deploys on main/dev branch pushes (not PRs)
      # Note: continue-on-error prevents deployment failures from failing the tests badge
      - name: Deploy coverage to GitHub Pages
        continue-on-error: true
        if: |
          always() &&
          github.ref == 'refs/heads/main' &&
          github.event_name == 'push'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -ex

          # Check source exists
          if [ ! -d "coverage/htmlcov" ]; then
            echo "ERROR: coverage/htmlcov does not exist"
            exit 1
          fi

          FILE_COUNT=$(find coverage/htmlcov -type f | wc -l)
          echo "Found $FILE_COUNT files to deploy"

          if [ "$FILE_COUNT" -eq 0 ]; then
            echo "ERROR: No files to deploy"
            exit 1
          fi

          # Copy to a fresh directory to avoid Docker volume/git issues
          DEPLOY_DIR=$(mktemp -d)
          cp -r coverage/htmlcov/* "$DEPLOY_DIR/"
          cd "$DEPLOY_DIR"

          # Create .nojekyll to bypass Jekyll processing
          touch .nojekyll

          # Debug: show what we have
          echo "Files in deploy dir:"
          find . -type f | head -10
          echo "Total files: $(find . -type f | wc -l)"

          # Setup git
          git init
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Add all files and commit
          git add -A
          git status
          git commit -m "docs: update coverage report ${{ github.sha }}"

          # Force push to gh-pages branch
          git push --force "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" HEAD:gh-pages

          echo "Deployment complete!"
          rm -rf "$DEPLOY_DIR"

      # Add coverage summary as PR comment for easy visibility
      - name: Comment coverage on PR
        if: always() && github.event_name == 'pull_request'
        continue-on-error: true  # Don't fail the job if GitHub API is throttled
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          retries: 3
          script: |
            const fs = require('fs');

            let lineCoverage = 'N/A';
            let branchCoverage = 'N/A';
            let linesCovered = 0;
            let linesTotal = 0;
            let filesAnalyzed = 0;
            let lowestCoverageFiles = [];

            try {
              if (fs.existsSync('coverage/coverage.xml')) {
                const xml = fs.readFileSync('coverage/coverage.xml', 'utf8');

                // Extract overall metrics
                const lineMatch = xml.match(/line-rate="([^"]+)"/);
                const branchMatch = xml.match(/branch-rate="([^"]+)"/);
                const coveredMatch = xml.match(/lines-covered="([^"]+)"/);
                const validMatch = xml.match(/lines-valid="([^"]+)"/);

                if (lineMatch) lineCoverage = (parseFloat(lineMatch[1]) * 100).toFixed(1) + '%';
                if (branchMatch) branchCoverage = (parseFloat(branchMatch[1]) * 100).toFixed(1) + '%';
                if (coveredMatch) linesCovered = parseInt(coveredMatch[1]);
                if (validMatch) linesTotal = parseInt(validMatch[1]);

                // Extract per-file coverage to find lowest coverage files
                const classMatches = xml.matchAll(/<class[^>]+filename="([^"]+)"[^>]+line-rate="([^"]+)"/g);
                const fileCoverages = [];
                for (const match of classMatches) {
                  const filename = match[1].replace(/^src\//, '').replace(/^local_deep_research\//, '');
                  const rate = parseFloat(match[2]) * 100;
                  fileCoverages.push({ filename, rate });
                  filesAnalyzed++;
                }

                // Sort by coverage (lowest first) and get bottom 5 with <50% coverage
                lowestCoverageFiles = fileCoverages
                  .filter(f => f.rate < 50)
                  .sort((a, b) => a.rate - b.rate)
                  .slice(0, 5);
              }
            } catch (e) {
              console.log('Could not parse coverage:', e);
            }

            // Build the lowest coverage files section
            let lowestFilesSection = '';
            if (lowestCoverageFiles.length > 0) {
              lowestFilesSection = '\n**Files needing attention** (<50% coverage):\n' +
                lowestCoverageFiles.map(f => `- \`${f.filename}\`: ${f.rate.toFixed(1)}%`).join('\n');
            }

            const body = `## 📊 Coverage Report

            | Metric | Value |
            |--------|-------|
            | **Line Coverage** | ${lineCoverage} |
            | **Branch Coverage** | ${branchCoverage} |
            | **Lines** | ${linesCovered.toLocaleString()} / ${linesTotal.toLocaleString()} |
            | **Files Analyzed** | ${filesAnalyzed} |

            📈 [View Full Report](${{ env.PAGES_URL }}) _(updates after merge)_

            <details>
            <summary>📉 Coverage Details</summary>
            ${lowestFilesSection || '\n✅ All files have >50% coverage!'}

            ---
            - Coverage is calculated from \`src/\` directory
            - Full interactive HTML report available after merge to main/dev
            - Download artifacts for immediate detailed view

            </details>`;

            // Find existing comment. Paginate — on a long-lived PR the coverage
            // comment may be past the first page (listComments defaults to 30),
            // which would otherwise create a new comment every run.
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              per_page: 100,
            });

            const botComment = comments.find(c =>
              c.user.type === 'Bot' && c.body.includes('📊 Coverage Report')
            );

            if (botComment) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: body
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: body
              });
            }

      # Coverage badge via GitHub Gist + shields.io (no third-party service needed)
      # How it works:
      # 1. We store a JSON file in a GitHub Gist (gist.github.com - GitHub's snippet/paste service)
      # 2. shields.io reads that JSON to generate a badge image
      # 3. README references the shields.io URL to display the badge
      # Setup: Create a gist, add GIST_TOKEN (PAT with gist scope) and COVERAGE_GIST_ID secrets
      - name: Update coverage badge
        if: always() && github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          GIST_TOKEN: ${{ secrets.GIST_TOKEN }}
        run: |
          if [ -z "$GIST_TOKEN" ]; then
            echo "GIST_TOKEN not set, skipping badge update"
            exit 0
          fi
          if [ -f coverage/coverage.xml ]; then
            COVERAGE=$(python3 -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage/coverage.xml'); print(f\"{float(tree.getroot().get('line-rate', 0)) * 100:.0f}\")" 2>/dev/null || echo "0")
            # Determine color based on coverage
            if [ "$COVERAGE" -ge 80 ]; then
              COLOR="brightgreen"
            elif [ "$COVERAGE" -ge 60 ]; then
              COLOR="green"
            elif [ "$COVERAGE" -ge 40 ]; then
              COLOR="yellow"
            else
              COLOR="red"
            fi
            # Create badge JSON
            echo "{\"schemaVersion\":1,\"label\":\"coverage\",\"message\":\"${COVERAGE}%\",\"color\":\"${COLOR}\"}" > coverage-badge.json
            # Update gist (requires GIST_TOKEN secret and GIST_ID)
            if [ -n "${{ secrets.COVERAGE_GIST_ID }}" ]; then
              curl -s -X PATCH \
                -H "Authorization: token ${GIST_TOKEN}" \
                -H "Accept: application/vnd.github.v3+json" \
                "https://api.github.com/gists/${{ secrets.COVERAGE_GIST_ID }}" \
                -d "{\"files\":{\"coverage-badge.json\":{\"content\":$(jq -Rs . < coverage-badge.json)}}}"
            fi
          fi

  # Run UI tests with Puppeteer
  #
  # Runs as a 17-way matrix (one cell per shard). Each cell spins up its own
  # docker container and runs only the tests tagged with its shard in
  # tests/ui_tests/run_all_tests.js. Fresh per-cell DB bounds state leakage
  # across tests (see #5165 flake analysis).
  #
  # Shard design: max 4 tests per shard to prevent cascade failures from
  # server starvation. Heavy tests (e.g., auth-register with SQLCipher DB
  # creation) are isolated into dedicated shards.
  #
  # Keep in sync with VALID_SHARDS in tests/ui_tests/run_all_tests.js.
  #
  # Trigger: only in the release pipeline (workflow_call with strict-mode=true).
  # Skipped on every PR and push for fast developer feedback. The release gate
  # is where UI regressions actually get caught and fixed.
  ui-tests:
    runs-on: ubuntu-latest
    name: UI Tests (Puppeteer) [${{ matrix.shard }}]
    needs: build-test-image
    if: ${{ inputs.strict-mode == true }}
    # 3 attempts × 12-min per-attempt cap + cleanup + outer setup steps =
    # ~40 min absolute worst case. 45 gives the retry wrapper room to capture
    # final logs even if every attempt hits its cap.
    timeout-minutes: 45
    strategy:
      matrix:
        shard:
          - auth-login
          - auth-register
          - auth-pages
          - research-workflow
          - research-form
          - research-metrics
          - settings-core
          - settings-pages
          - library
          - history-news
          - mobile
          - api-crud
          - error-benchmark
          - accessibility
          - chat-core
          - chat-lifecycle
          - link-analytics
      fail-fast: false
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      - name: Install Node.js for UI tests
        # zizmor: ignore[cache-poisoning] - caching explicitly disabled with cache: ''
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: '24'
          # Cache poisoning protection: explicitly disable caching on PR events from forks
          cache: ''

      - name: Install UI test dependencies
        run: |
          export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
          cd tests && npm ci

      # Combined "init DB → start server → register user → run tests" in a
      # retry block so each attempt gets fresh server + DB state, equivalent
      # to GitHub's "Re-run failed jobs" but skipping the docker artifact
      # re-download (the tarball is bit-identical between attempts).
      #
      # 3 attempts because 17 independent shards each at ~98% success only
      # gives ~75% gate pass; 2 retries brings per-shard fail to ~0.0001%.
      # More than 3 starts hiding genuine bugs (a test that fails 3/3 is a
      # regression, not a flake).
      - name: Run UI tests with retry (fresh server per attempt)
        uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
        env:
          SHARD: ${{ matrix.shard }}
          TEST_IMAGE: ${{ env.TEST_IMAGE }}
        with:
          timeout_minutes: 12
          max_attempts: 3
          retry_on: error
          shell: bash
          # Runs after a failed attempt, before the next. Snapshots the
          # server log under a numbered filename so failure forensics
          # survive even when a later attempt passes.
          on_retry_command: |
            mkdir -p /tmp/ldr-ci-logs
            ATTEMPT=$(( $(find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${SHARD}-attempt-*.log" 2>/dev/null | wc -l) + 1 ))
            echo "::group::Cleanup after failed attempt ${ATTEMPT}"
            docker logs ldr-server > "/tmp/ldr-ci-logs/server-${SHARD}-attempt-${ATTEMPT}.log" 2>&1 || true
            docker stop ldr-server || true
            docker rm ldr-server || true
            sudo rm -rf "${GITHUB_WORKSPACE}/ci-data" || true
            echo "::endgroup::"
          command: |
            set -e
            mkdir -p "${GITHUB_WORKSPACE}/ci-data"

            echo "::group::Initialize test database"
            docker run --rm \
              -v "${GITHUB_WORKSPACE}":/app \
              -v "${GITHUB_WORKSPACE}/ci-data":/data \
              -e TEST_ENV=true \
              -e LDR_DATA_DIR=/data \
              -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
              -e LDR_TEST_MODE=1 \
              -w /app/src \
              "${TEST_IMAGE}" \
              python ../scripts/ci/init_test_database.py
            echo "::endgroup::"

            # The link-analytics shard renders /metrics/links, whose domain
            # cards come entirely from ResearchResource rows. A fresh DB has
            # none, so seed a small fixture for this shard ONLY — other shards
            # (and the empty-state screenshot suites) keep an empty DB.
            if [ "${SHARD}" = "link-analytics" ]; then
              echo "::group::Seed link analytics data"
              docker run --rm \
                -v "${GITHUB_WORKSPACE}":/app \
                -v "${GITHUB_WORKSPACE}/ci-data":/data \
                -e TEST_ENV=true \
                -e LDR_DATA_DIR=/data \
                -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
                -e LDR_TEST_MODE=1 \
                -w /app/src \
                "${TEST_IMAGE}" \
                python ../scripts/ci/seed_link_analytics.py
              echo "::endgroup::"
            fi

            echo "::group::Start application server"
            docker run -d \
              --name ldr-server \
              -p 5000:5000 \
              -v "${GITHUB_WORKSPACE}/ci-data":/data \
              -e CI=true \
              -e TEST_ENV=true \
              -e FLASK_ENV=testing \
              -e LDR_DISABLE_RATE_LIMITING=true \
              -e SECRET_KEY=test-secret-key-for-ci \
              -e LDR_DATA_DIR=/data \
              -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
              -e LDR_TEST_MODE=1 \
              "${TEST_IMAGE}" ldr-web

            # --max-time bounds total request time; --connect-timeout bounds
            # TCP handshake. Without these, a hung connection would silently
            # eat the 12-min per-attempt budget instead of failing fast.
            for i in {1..60}; do
              if curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then
                echo "Server is ready after $i seconds"
                break
              fi
              if ! docker ps --filter "name=ldr-server" --filter "status=running" -q | grep -q .; then
                echo "Server container died!"
                echo "Server log:"
                docker logs ldr-server
                exit 1
              fi
              echo "Waiting for server... ($i/60)"
              sleep 1
            done

            if ! curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then
              echo "Server failed to start after 60 seconds"
              echo "Server log:"
              docker logs ldr-server
              exit 1
            fi
            echo "::endgroup::"

            echo "::group::Register CI test user"
            docker run --rm \
              -v "${GITHUB_WORKSPACE}":/app \
              -e CI=true \
              -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
              -e LDR_TEST_MODE=1 \
              --network host \
              -w /app/tests/ui_tests \
              "${TEST_IMAGE}" \
              node register_ci_user.js http://localhost:5000
            echo "::endgroup::"

            echo "::group::Run UI tests for shard ${SHARD}"
            docker run --rm \
              -v "${GITHUB_WORKSPACE}":/app \
              -e CI=true \
              -e PUPPETEER_HEADLESS=true \
              -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
              -e LDR_TEST_MODE=1 \
              --network host \
              -w /app/tests/ui_tests \
              "${TEST_IMAGE}" \
              sh -c "xvfb-run -a -s '-screen 0 1920x1080x24' node run_all_tests.js --shard=${SHARD}"
            echo "::endgroup::"

      - name: Capture final server logs on failure
        if: failure()
        run: |
          mkdir -p /tmp/ldr-ci-logs
          ATTEMPT=$(( $(find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${{ matrix.shard }}-attempt-*.log" 2>/dev/null | wc -l) + 1 ))
          FINAL_LOG="/tmp/ldr-ci-logs/server-${{ matrix.shard }}-attempt-${ATTEMPT}-final.log"
          docker logs ldr-server > "$FINAL_LOG" 2>&1 || true
          {
            echo "## 🪵 ldr-server logs [${{ matrix.shard }}] — final attempt (last 200 lines)"
            echo ""
            echo '```'
            tail -n 200 "$FINAL_LOG" || true
            echo '```'
            echo ""
            echo "Grepping for SQLAlchemy pool / DB session issues across all attempts:"
            echo '```'
            grep -hE "QueuePool|connection timed out|TimeoutError|cleanup_dead_threads|Failed to open database|DatabaseSessionError" /tmp/ldr-ci-logs/server-"${{ matrix.shard }}"-attempt-*.log 2>/dev/null | tail -n 50 || true
            echo '```'
            echo ""
            echo "Attempt logs preserved for upload:"
            find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${{ matrix.shard }}-attempt-*.log" -printf '%TY-%Tm-%Td %TH:%TM  %s\t%p\n' 2>/dev/null || true
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Stop server
        if: always()
        run: |
          docker stop ldr-server || true
          docker rm ldr-server || true

      - name: Upload UI test screenshots
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ui-test-screenshots-${{ matrix.shard }}
          path: |
            tests/screenshots/
            tests/ui_tests/screenshots/

      - name: Upload server logs on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ui-tests-server-log-${{ matrix.shard }}
          path: /tmp/ldr-ci-logs/server-${{ matrix.shard }}-attempt-*.log
          retention-days: 3
          if-no-files-found: ignore

  # Aggregates matrix-cell results into a single stable check name.
  # ci-gate.yml and any branch-protection rule should depend on this job's
  # name ("UI Tests (Puppeteer)"), not on individual shard cells.
  ui-tests-summary:
    runs-on: ubuntu-latest
    name: UI Tests (Puppeteer)
    needs: ui-tests
    if: ${{ always() && inputs.strict-mode == true }}
    permissions:
      contents: read
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Check all shards succeeded
        run: |
          result="${{ needs.ui-tests.result }}"
          echo "Aggregated matrix result: $result"
          if [ "$result" != "success" ]; then
            echo "::error::One or more UI test shards did not succeed (result=$result)."
            exit 1
          fi
          echo "All UI test shards passed."

  # Run accessibility tests (Playwright a11y + Lighthouse)
  accessibility-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    name: Accessibility Tests
    needs: [build-test-image, detect-changes]
    if: needs.detect-changes.outputs.accessibility == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      - name: Initialize test database
        run: |
          mkdir -p "$PWD/ci-data"
          docker run --rm \
            -v "$PWD":/app \
            -v "$PWD/ci-data":/data \
            -e TEST_ENV=true \
            -e LDR_DATA_DIR=/data \
            -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
            -e LDR_TEST_MODE=1 \
            -w /app/src \
            ${{ env.TEST_IMAGE }} \
            python ../scripts/ci/init_test_database.py

      - name: Start application server in Docker
        run: |
          docker run -d \
            --name ldr-a11y-server \
            -p 5000:5000 \
            -v "$PWD/ci-data":/data \
            -e CI=true \
            -e TEST_ENV=true \
            -e FLASK_ENV=testing \
            -e LDR_DISABLE_RATE_LIMITING=true \
            -e SECRET_KEY=test-secret-key-for-ci \
            -e LDR_DATA_DIR=/data \
            -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
            -e LDR_TEST_MODE=1 \
            ${{ env.TEST_IMAGE }} ldr-web

          # Wait for server to be ready.
          # --connect-timeout/--max-time bound TCP and total request time so a
          # hung connection fails fast instead of eating the job timeout.
          for i in {1..60}; do
            if curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then
              echo "Server is ready after $i seconds"
              break
            fi
            if ! docker ps --filter "name=ldr-a11y-server" --filter "status=running" -q | grep -q .; then
              echo "Server container died!"
              echo "Server log:"
              docker logs ldr-a11y-server
              exit 1
            fi
            echo "Waiting for server... ($i/60)"
            sleep 1
          done

          # Final check
          if ! curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then
            echo "Server failed to start after 60 seconds"
            echo "Server log:"
            docker logs ldr-a11y-server
            exit 1
          fi

      - name: Register CI test user
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
            -e LDR_TEST_MODE=1 \
            -e NODE_PATH=/install/tests/ui_tests/node_modules \
            --network host \
            -w /app/tests/ui_tests \
            ${{ env.TEST_IMAGE }} \
            node register_ci_user.js http://localhost:5000

      - name: Run Playwright accessibility tests
        timeout-minutes: 5
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            -e BASE_URL=http://localhost:5000 \
            -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \
            -e LDR_TEST_MODE=1 \
            --network host \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            bash -c '\
              ln -sfn /install/tests/accessibility_tests/node_modules /app/node_modules && \
              ln -sfn /install/tests/accessibility_tests/node_modules /app/tests/accessibility_tests/node_modules && \
              /install/tests/accessibility_tests/node_modules/.bin/playwright test \
                --config=/app/playwright.config.js --project=chromium; \
              EXIT=$?; \
              rm -f /app/node_modules /app/tests/accessibility_tests/node_modules; \
              exit $EXIT'

      - name: Run Lighthouse accessibility audit
        if: always()
        timeout-minutes: 5
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e CI=true \
            --network host \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            bash -c '\
              if [ ! -x /usr/local/bin/chrome ]; then echo "ERROR: Chrome not found at /usr/local/bin/chrome" >&2; exit 1; fi && \
              echo "#!/bin/bash" > /tmp/chrome-wrapper.sh && \
              echo "exec /usr/local/bin/chrome --no-sandbox --headless \"\$@\"" >> /tmp/chrome-wrapper.sh && \
              chmod +x /tmp/chrome-wrapper.sh && \
              CHROME_PATH=/tmp/chrome-wrapper.sh \
              /install/tests/accessibility_tests/node_modules/.bin/lhci autorun --config=/app/lighthouserc.json'

      - name: Capture server logs on failure
        if: failure()
        run: |
          mkdir -p /tmp/ldr-ci-logs
          LOG_FILE=/tmp/ldr-ci-logs/ldr-a11y-server.log
          docker logs ldr-a11y-server > "$LOG_FILE" 2>&1 || true
          {
            echo "## 🪵 ldr-a11y-server logs (last 200 lines)"
            echo ""
            echo '```'
            tail -n 200 "$LOG_FILE" || true
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload accessibility reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: accessibility-reports
          path: |
            tests/accessibility_tests/playwright-report/
            tests/accessibility_tests/test-results/
            tests/accessibility_tests/.lighthouseci/

      - name: Upload server logs on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: a11y-server-log
          path: /tmp/ldr-ci-logs/ldr-a11y-server.log
          retention-days: 3
          if-no-files-found: ignore

      - name: Stop server
        if: always()
        run: |
          docker stop ldr-a11y-server || true
          docker rm ldr-a11y-server || true

  # Run LLM unit tests (conditional on path changes)
  llm-unit-tests:
    runs-on: ubuntu-latest
    name: LLM Unit Tests
    needs: [build-test-image, detect-changes]
    if: needs.detect-changes.outputs.llm == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      - name: Run LLM registry tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/test_llm/test_llm_registry.py -v -n auto"

      - name: Run LLM integration tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/test_llm/test_llm_integration.py -v -n auto"

      - name: Run API LLM integration tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/test_llm/test_api_llm_integration.py -v -n auto"

      - name: Run LLM edge case tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/test_llm/test_llm_edge_cases.py -v -n auto"

      - name: Run LLM benchmark tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "python -m pytest tests/test_llm/test_llm_benchmarks.py -v -n auto"

  # Run LLM example tests (conditional on path changes)
  llm-example-tests:
    runs-on: ubuntu-latest
    name: LLM Example Tests
    needs: [build-test-image, detect-changes]
    if: needs.detect-changes.outputs.llm == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
    timeout-minutes: 20
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      # basic_custom_llm.py and advanced_custom_llm.py are intentionally NOT
      # executed in CI: they run real research pipelines against Wikipedia and
      # PubMed, which makes them flaky under a 60s budget. The examples are
      # kept in the repo as documentation and are covered by the compile check
      # below; mock_llm_example.py exercises the same integration path offline.
      - name: Test mock LLM example
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "timeout 60s python examples/llm_integration/mock_llm_example.py"

      - name: Test direct import
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "timeout 60s python examples/api_usage/programmatic/test_direct_import.py"

      - name: API public contract guardrail
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "timeout 60s python examples/api_usage/programmatic/api_public_contract_guardrail.py"

      - name: Compile check all remaining examples
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -e PYTHONPATH=/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "
              python -m py_compile examples/api_usage/programmatic/simple_programmatic_example.py &&
              python -m py_compile examples/api_usage/programmatic/advanced_features_example.py &&
              python -m py_compile examples/api_usage/programmatic/search_strategies_example.py &&
              python -m py_compile examples/api_usage/programmatic/minimal_working_example.py &&
              python -m py_compile examples/api_usage/programmatic/searxng_example.py &&
              python -m py_compile examples/api_usage/programmatic/custom_llm_retriever_example.py &&
              python -m py_compile examples/api_usage/programmatic/hybrid_search_example.py &&
              python -m py_compile examples/llm_integration/basic_custom_llm.py &&
              python -m py_compile examples/llm_integration/advanced_custom_llm.py &&
              python -m py_compile examples/api_usage/http/simple_working_example.py &&
              python -m py_compile examples/api_usage/http/advanced/http_api_examples.py &&
              python -m py_compile examples/api_usage/http/advanced/simple_http_example.py &&
              python -m py_compile examples/api_usage/simple_client_example.py &&
              python -m py_compile examples/example_browsecomp.py &&
              python -m py_compile examples/show_env_vars.py &&
              python -m py_compile examples/run_benchmark.py &&
              python -m py_compile examples/elasticsearch/search_example.py &&
              echo 'All compile checks passed'
            "

  # Smoke test for the production ldr image
  ldr-production-smoke-test:
    runs-on: ubuntu-latest
    name: Production Image Smoke Test
    needs: detect-changes
    if: needs.detect-changes.outputs.docker == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
    timeout-minutes: 30
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: Build production ldr image
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
        with:
          context: .
          target: ldr
          load: true
          tags: ldr-prod
          # Cache poisoning protection: only read/write cache on trusted events (not PRs from forks)
          cache-from: ${{ github.event_name != 'pull_request' && 'type=gha,scope=ldr-prod' || '' }}
          cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=ldr-prod' || '' }}

      - name: Verify file ownership and playwright access
        run: |
          docker run --rm ldr-prod sh -c '
            echo "Checking /install/.venv ownership..."
            VENV_OWNER=$(stat -c "%U:%G" /install/.venv)
            if [ "$VENV_OWNER" != "ldruser:ldruser" ]; then
              echo "FAIL: /install/.venv owned by $VENV_OWNER, expected ldruser:ldruser"
              exit 1
            fi
            echo "OK: /install/.venv owned by $VENV_OWNER"

            echo "Spot-checking venv file ownership..."
            BAD_FILES=$(find /install/.venv \( ! -user ldruser -o ! -group ldruser \) -print -quit)
            if [ -n "$BAD_FILES" ]; then
              echo "FAIL: files not owned by ldruser:ldruser:"
              find /install/.venv \( ! -user ldruser -o ! -group ldruser \) | head -10
              exit 1
            fi
            echo "OK: venv files owned by ldruser:ldruser"

            echo "Checking /install parent dir ownership..."
            INSTALL_OWNER=$(stat -c "%U:%G" /install)
            if [ "$INSTALL_OWNER" != "ldruser:ldruser" ]; then
              echo "FAIL: /install owned by $INSTALL_OWNER, expected ldruser:ldruser"
              exit 1
            fi
            echo "OK: /install owned by $INSTALL_OWNER"

            echo "Checking /scripts parent dir ownership..."
            SCRIPTS_OWNER=$(stat -c "%U:%G" /scripts)
            if [ "$SCRIPTS_OWNER" != "ldruser:ldruser" ]; then
              echo "FAIL: /scripts owned by $SCRIPTS_OWNER, expected ldruser:ldruser"
              exit 1
            fi
            echo "OK: /scripts owned by $SCRIPTS_OWNER"

            echo "Checking /home/ldruser ownership..."
            HOME_OWNER=$(stat -c "%U:%G" /home/ldruser)
            if [ "$HOME_OWNER" != "ldruser:ldruser" ]; then
              echo "FAIL: /home/ldruser owned by $HOME_OWNER, expected ldruser:ldruser"
              exit 1
            fi
            echo "OK: /home/ldruser owned by $HOME_OWNER"

            echo "Checking playwright is accessible as ldruser..."
            playwright --version
            echo "OK: playwright accessible as ldruser"

            echo "Checking playwright browsers are NOT in production image..."
            if [ -d /home/ldruser/.cache/ms-playwright ]; then
              echo "FAIL: Playwright browsers found in production image (should only be in ldr-test stage)"
              exit 1
            fi
            echo "OK: no playwright browsers in production image (as expected)"
          '

      - name: Start production container
        run: |
          mkdir -p "$PWD/ci-data-prod"
          # Run with the same security settings as docker-compose.yml so CI
          # catches capability regressions (e.g., missing cap_add that breaks
          # the entrypoint's chown/setpriv in restricted environments).
          docker run -d \
            --name ldr-prod-test \
            -p 5001:5000 \
            -v "$PWD/ci-data-prod":/data \
            --security-opt no-new-privileges:true \
            --cap-drop ALL \
            --cap-add CHOWN \
            --cap-add FOWNER \
            --cap-add DAC_OVERRIDE \
            --cap-add SETUID \
            --cap-add SETGID \
            -e LDR_DATA_DIR=/data \
            -e LDR_DISABLE_RATE_LIMITING=true \
            -e SECRET_KEY=test-secret-key-for-ci \
            ldr-prod

      - name: Wait for healthy status
        run: |
          echo "Waiting for server to be healthy..."
          for i in {1..60}; do
            if curl -sf --connect-timeout 2 --max-time 5 http://localhost:5001/api/v1/health > /dev/null 2>&1; then
              echo "Server is healthy after $i seconds"
              exit 0
            fi
            if ! docker ps --filter "name=ldr-prod-test" --filter "status=running" -q | grep -q .; then
              echo "Container died!"
              docker logs ldr-prod-test
              exit 1
            fi
            echo "Waiting... ($i/60)"
            sleep 1
          done
          echo "Server failed to become healthy"
          docker logs ldr-prod-test
          exit 1

      - name: Stop container
        if: always()
        run: |
          docker stop ldr-prod-test || true
          docker rm ldr-prod-test || true

  # Run infrastructure tests (conditional on path changes)
  infrastructure-tests:
    runs-on: ubuntu-latest
    name: Infrastructure Tests
    needs: [build-test-image, detect-changes]
    if: needs.detect-changes.outputs.infrastructure == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true
    permissions:
      contents: read

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Download Docker image
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: docker-image
          path: /tmp

      - name: Load Docker image
        run: docker load --input /tmp/ldr-test.tar

      - name: Set up Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: '24'

      - name: Install JavaScript test dependencies
        run: |
          cd tests/infrastructure_tests && npm ci

      - name: Run Python infrastructure tests
        run: |
          docker run --rm \
            -v "$PWD":/app \
            -w /app \
            ${{ env.TEST_IMAGE }} \
            sh -c "cd /app && python -m pytest tests/infrastructure_tests/test_*.py -v --color=yes --no-header -rN"

      - name: Run JavaScript infrastructure tests
        run: |
          cd tests/infrastructure_tests && npm test

      - name: Run Vitest JS unit tests
        run: |
          npm ci && npm test