name: Docker Tests (Consolidated) on: pull_request: types: [opened, synchronize, reopened] branches: [ main ] push: branches: [ main ] workflow_call: # Called by ci-gate.yml for release pipeline inputs: strict-mode: description: 'Run in strict mode (all tests blocking, no continue-on-error)' required: false type: boolean default: true secrets: OPENROUTER_API_KEY: required: false GIST_TOKEN: required: false COVERAGE_GIST_ID: required: false workflow_dispatch: # No concurrency group ā€” intentionally omitted. # This workflow triggers on both pull_request and workflow_call (from # ci-gate.yml / release-gate.yml). A shared concurrency key would cause # direct PR runs and workflow_call runs to cancel each other mid-flight. # See #3554 (reverted in #3599) for context. # Top-level permissions set to minimum (OSSF Scorecard Token-Permissions) permissions: {} env: TEST_IMAGE: ldr-test PAGES_URL: https://learningcircuit.github.io/local-deep-research/ jobs: # Detect which paths changed to conditionally run certain jobs detect-changes: runs-on: ubuntu-latest outputs: llm: ${{ steps.filter.outputs.llm }} infrastructure: ${{ steps.filter.outputs.infrastructure }} docker: ${{ steps.filter.outputs.docker }} accessibility: ${{ steps.filter.outputs.accessibility }} steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | llm: - 'src/local_deep_research/llm/**' - 'src/local_deep_research/config/llm_config.py' - 'src/local_deep_research/api/research_functions.py' - 'tests/test_llm/**' - 'examples/**' - '.github/workflows/docker-tests.yml' infrastructure: - 'src/local_deep_research/web/routes/**' - 'src/local_deep_research/web/static/js/**' - 'tests/infrastructure_tests/**' - 'tests/js/**' - '.github/workflows/docker-tests.yml' docker: - 'Dockerfile' - '.dockerignore' - 'scripts/ldr_entrypoint.sh' - 'scripts/ollama_entrypoint.sh' - 'pyproject.toml' - 'pdm.lock' - 'package.json' - 'package-lock.json' - 'vite.config.js' - '.github/workflows/docker-tests.yml' accessibility: - 'src/local_deep_research/web/templates/**' - 'src/local_deep_research/web/static/css/**' - 'src/local_deep_research/web/static/js/**' - 'tests/accessibility_tests/**' - 'playwright.config.js' - 'lighthouserc.json' - '.github/workflows/docker-tests.yml' # Build Docker image once and share via artifact build-test-image: runs-on: ubuntu-latest name: Build Test Image if: | github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || inputs.strict-mode == true steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build Docker image with cache uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 with: context: . target: ldr-test outputs: type=docker,dest=/tmp/ldr-test.tar tags: ${{ env.TEST_IMAGE }} # Cache poisoning protection: only read/write cache on trusted events (not PRs from forks) cache-from: ${{ github.event_name != 'pull_request' && 'type=gha,scope=ldr-test' || '' }} cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=ldr-test' || '' }} - name: Upload Docker image as artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: docker-image path: /tmp/ldr-test.tar retention-days: 1 # Run all pytest tests with coverage pytest-tests: runs-on: ubuntu-latest name: All Pytest Tests + Coverage # Cap exists so a hung run doesn't inherit GitHub's 360-minute default # (observed 2026-06-10 on PR #4491: a flaked run sat in the main pytest # step for 2+ hours before being cancelled manually). The original # 75-min cap (#4514) was sized from a 19-57 min historical range, but # healthy runs legitimately reach ~95 min (e.g. PR #4465 on 2026-06-07: # 1h35m in the pytest step alone; #4471/#4517 were killed mid-suite at # 75 min the day the cap merged). 120 min = worst observed healthy run # + ~25% headroom, while still ending a truly hung run 4x sooner than # the default. timeout-minutes: 120 needs: build-test-image environment: ci permissions: contents: write # Needed for gh-pages deployment pull-requests: write # Needed for PR comments steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: true # Needed for gh-pages push - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar - name: Set up Node.js # zizmor: ignore[cache-poisoning] - caching explicitly disabled with cache: '' uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '24' # Cache poisoning protection: explicitly disable caching on PR events from forks cache: '' - name: Install infrastructure test dependencies run: | cd tests/infrastructure_tests && npm ci - name: Run all pytest tests with coverage # Tests are always blocking - failures should not be silently ignored. # test_settings_routes_coverage.py is excluded here and run serially # in a dedicated step below ā€” its 94 authenticated_client tests # (each one: register + login + SQLCipher KDF + 10 Alembic migrations) # accumulate enough connection/FD pressure inside the assigned xdist # worker to deadlock it silently late in the run (the worker dies # without a timeout, its coverage data is lost, total coverage drops # under the 50% fail-under gate, the job fails). Same pattern as # fd_canary tests, same fix: serial, no '-n auto' for this file. env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | mkdir -p coverage docker run --rm \ -v "$PWD":/app \ -e CI=true \ -e GITHUB_ACTIONS=true \ -e LDR_TESTING_WITH_MOCKS=true \ -e LDR_DISABLE_RATE_LIMITING=true \ -e PYTHONPATH=/app/src \ -e OPENROUTER_API_KEY="${OPENROUTER_API_KEY}" \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/ \ --cov=src \ --cov-report= \ --cov-fail-under=0 \ --tb=short \ --no-header \ -q \ --durations=20 \ -n auto \ -m 'not integration and not fd_canary' \ --ignore=tests/ui_tests \ --ignore=tests/infrastructure_tests \ --ignore=tests/api_tests_with_login \ --ignore=tests/fuzz \ --ignore=tests/web/routes/test_settings_routes_coverage.py" # See the comment on the main pytest step above for why this file is # split out. Runs SERIAL (no '-n auto'), appends coverage data into # the main run's .coverage file, and emits the final xml + html # reports + the 50% fail-under check after both runs have contributed. - name: Run test_settings_routes_coverage.py (serial, with coverage) env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | docker run --rm \ -v "$PWD":/app \ -e CI=true \ -e GITHUB_ACTIONS=true \ -e LDR_TESTING_WITH_MOCKS=true \ -e LDR_DISABLE_RATE_LIMITING=true \ -e PYTHONPATH=/app/src \ -e OPENROUTER_API_KEY="${OPENROUTER_API_KEY}" \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/web/routes/test_settings_routes_coverage.py \ --cov=src \ --cov-append \ --cov-report=xml:coverage/coverage.xml \ --cov-report=html:coverage/htmlcov \ --cov-fail-under=50 \ --tb=short \ --no-header \ -q \ --durations=20" # fd_canary tests (#3816 FD-leak guards) are CPU-bound real-event-loop # spins. Under coverage tracing + '-n auto' contention they blow past # the global 60s timeout and pressure xdist workers into crashing. # Run them in a dedicated SERIAL step with NO coverage and NO '-n auto' # so the FD-count signal stays deterministic (~10s) without flaking. - name: Run fd_canary FD-leak tests (serial, no coverage) if: always() run: | docker run --rm \ -v "$PWD":/app \ -e CI=true \ -e GITHUB_ACTIONS=true \ -e LDR_TESTING_WITH_MOCKS=true \ -e LDR_DISABLE_RATE_LIMITING=true \ -e PYTHONPATH=/app/src \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/utilities/test_close_base_llm.py \ -m fd_canary \ -p no:cov \ --tb=short \ --no-header \ -q" - name: Run JavaScript infrastructure tests if: always() run: | cd tests/infrastructure_tests && npm test - name: Run Vitest JS unit tests if: always() run: | npm ci && npm test - name: Generate coverage summary if: always() run: | { echo "## Coverage Report" echo "" if [ -f coverage/coverage.xml ]; then # Extract total coverage from XML COVERAGE=$(python3 -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage/coverage.xml'); print(f\"{float(tree.getroot().get('line-rate', 0)) * 100:.1f}%\")" 2>/dev/null || echo "N/A") echo "**Total Line Coverage: ${COVERAGE}**" echo "" echo "šŸ“Š **[View Full Coverage Report](${{ env.PAGES_URL }})** - Interactive HTML report with line-by-line details" echo "" echo "_Note: Report updates after merge to main/dev branch._" else echo "āš ļø Coverage report not generated" fi } >> "$GITHUB_STEP_SUMMARY" - name: Upload coverage reports if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-report path: coverage/ # Fix permissions on coverage directory (Docker creates files as root) - name: Fix coverage directory permissions if: always() run: | if [ -d "coverage" ]; then sudo chown -R "$(id -u):$(id -g)" coverage/ echo "Coverage directory owner after chown:" stat coverage/ if [ -d "coverage/htmlcov" ]; then echo "HTML coverage file count:" find coverage/htmlcov/ -type f | wc -l echo "Sample files:" find coverage/htmlcov/ -maxdepth 1 -type f -name "*.html" | head -5 else echo "ERROR: coverage/htmlcov directory does not exist!" fi else echo "ERROR: coverage directory does not exist!" fi # Deploy coverage HTML report to GitHub Pages # Only deploys on main/dev branch pushes (not PRs) # Note: continue-on-error prevents deployment failures from failing the tests badge - name: Deploy coverage to GitHub Pages continue-on-error: true if: | always() && github.ref == 'refs/heads/main' && github.event_name == 'push' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -ex # Check source exists if [ ! -d "coverage/htmlcov" ]; then echo "ERROR: coverage/htmlcov does not exist" exit 1 fi FILE_COUNT=$(find coverage/htmlcov -type f | wc -l) echo "Found $FILE_COUNT files to deploy" if [ "$FILE_COUNT" -eq 0 ]; then echo "ERROR: No files to deploy" exit 1 fi # Copy to a fresh directory to avoid Docker volume/git issues DEPLOY_DIR=$(mktemp -d) cp -r coverage/htmlcov/* "$DEPLOY_DIR/" cd "$DEPLOY_DIR" # Create .nojekyll to bypass Jekyll processing touch .nojekyll # Debug: show what we have echo "Files in deploy dir:" find . -type f | head -10 echo "Total files: $(find . -type f | wc -l)" # Setup git git init git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" # Add all files and commit git add -A git status git commit -m "docs: update coverage report ${{ github.sha }}" # Force push to gh-pages branch git push --force "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" HEAD:gh-pages echo "Deployment complete!" rm -rf "$DEPLOY_DIR" # Add coverage summary as PR comment for easy visibility - name: Comment coverage on PR if: always() && github.event_name == 'pull_request' continue-on-error: true # Don't fail the job if GitHub API is throttled uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: retries: 3 script: | const fs = require('fs'); let lineCoverage = 'N/A'; let branchCoverage = 'N/A'; let linesCovered = 0; let linesTotal = 0; let filesAnalyzed = 0; let lowestCoverageFiles = []; try { if (fs.existsSync('coverage/coverage.xml')) { const xml = fs.readFileSync('coverage/coverage.xml', 'utf8'); // Extract overall metrics const lineMatch = xml.match(/line-rate="([^"]+)"/); const branchMatch = xml.match(/branch-rate="([^"]+)"/); const coveredMatch = xml.match(/lines-covered="([^"]+)"/); const validMatch = xml.match(/lines-valid="([^"]+)"/); if (lineMatch) lineCoverage = (parseFloat(lineMatch[1]) * 100).toFixed(1) + '%'; if (branchMatch) branchCoverage = (parseFloat(branchMatch[1]) * 100).toFixed(1) + '%'; if (coveredMatch) linesCovered = parseInt(coveredMatch[1]); if (validMatch) linesTotal = parseInt(validMatch[1]); // Extract per-file coverage to find lowest coverage files const classMatches = xml.matchAll(/]+filename="([^"]+)"[^>]+line-rate="([^"]+)"/g); const fileCoverages = []; for (const match of classMatches) { const filename = match[1].replace(/^src\//, '').replace(/^local_deep_research\//, ''); const rate = parseFloat(match[2]) * 100; fileCoverages.push({ filename, rate }); filesAnalyzed++; } // Sort by coverage (lowest first) and get bottom 5 with <50% coverage lowestCoverageFiles = fileCoverages .filter(f => f.rate < 50) .sort((a, b) => a.rate - b.rate) .slice(0, 5); } } catch (e) { console.log('Could not parse coverage:', e); } // Build the lowest coverage files section let lowestFilesSection = ''; if (lowestCoverageFiles.length > 0) { lowestFilesSection = '\n**Files needing attention** (<50% coverage):\n' + lowestCoverageFiles.map(f => `- \`${f.filename}\`: ${f.rate.toFixed(1)}%`).join('\n'); } const body = `## šŸ“Š Coverage Report | Metric | Value | |--------|-------| | **Line Coverage** | ${lineCoverage} | | **Branch Coverage** | ${branchCoverage} | | **Lines** | ${linesCovered.toLocaleString()} / ${linesTotal.toLocaleString()} | | **Files Analyzed** | ${filesAnalyzed} | šŸ“ˆ [View Full Report](${{ env.PAGES_URL }}) _(updates after merge)_
šŸ“‰ Coverage Details ${lowestFilesSection || '\nāœ… All files have >50% coverage!'} --- - Coverage is calculated from \`src/\` directory - Full interactive HTML report available after merge to main/dev - Download artifacts for immediate detailed view
`; // Find existing comment. Paginate ā€” on a long-lived PR the coverage // comment may be past the first page (listComments defaults to 30), // which would otherwise create a new comment every run. const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, per_page: 100, }); const botComment = comments.find(c => c.user.type === 'Bot' && c.body.includes('šŸ“Š Coverage Report') ); if (botComment) { await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: botComment.id, body: body }); } else { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body: body }); } # Coverage badge via GitHub Gist + shields.io (no third-party service needed) # How it works: # 1. We store a JSON file in a GitHub Gist (gist.github.com - GitHub's snippet/paste service) # 2. shields.io reads that JSON to generate a badge image # 3. README references the shields.io URL to display the badge # Setup: Create a gist, add GIST_TOKEN (PAT with gist scope) and COVERAGE_GIST_ID secrets - name: Update coverage badge if: always() && github.ref == 'refs/heads/main' && github.event_name == 'push' env: GIST_TOKEN: ${{ secrets.GIST_TOKEN }} run: | if [ -z "$GIST_TOKEN" ]; then echo "GIST_TOKEN not set, skipping badge update" exit 0 fi if [ -f coverage/coverage.xml ]; then COVERAGE=$(python3 -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage/coverage.xml'); print(f\"{float(tree.getroot().get('line-rate', 0)) * 100:.0f}\")" 2>/dev/null || echo "0") # Determine color based on coverage if [ "$COVERAGE" -ge 80 ]; then COLOR="brightgreen" elif [ "$COVERAGE" -ge 60 ]; then COLOR="green" elif [ "$COVERAGE" -ge 40 ]; then COLOR="yellow" else COLOR="red" fi # Create badge JSON echo "{\"schemaVersion\":1,\"label\":\"coverage\",\"message\":\"${COVERAGE}%\",\"color\":\"${COLOR}\"}" > coverage-badge.json # Update gist (requires GIST_TOKEN secret and GIST_ID) if [ -n "${{ secrets.COVERAGE_GIST_ID }}" ]; then curl -s -X PATCH \ -H "Authorization: token ${GIST_TOKEN}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/gists/${{ secrets.COVERAGE_GIST_ID }}" \ -d "{\"files\":{\"coverage-badge.json\":{\"content\":$(jq -Rs . < coverage-badge.json)}}}" fi fi # Run UI tests with Puppeteer # # Runs as a 17-way matrix (one cell per shard). Each cell spins up its own # docker container and runs only the tests tagged with its shard in # tests/ui_tests/run_all_tests.js. Fresh per-cell DB bounds state leakage # across tests (see #5165 flake analysis). # # Shard design: max 4 tests per shard to prevent cascade failures from # server starvation. Heavy tests (e.g., auth-register with SQLCipher DB # creation) are isolated into dedicated shards. # # Keep in sync with VALID_SHARDS in tests/ui_tests/run_all_tests.js. # # Trigger: only in the release pipeline (workflow_call with strict-mode=true). # Skipped on every PR and push for fast developer feedback. The release gate # is where UI regressions actually get caught and fixed. ui-tests: runs-on: ubuntu-latest name: UI Tests (Puppeteer) [${{ matrix.shard }}] needs: build-test-image if: ${{ inputs.strict-mode == true }} # 3 attempts Ɨ 12-min per-attempt cap + cleanup + outer setup steps = # ~40 min absolute worst case. 45 gives the retry wrapper room to capture # final logs even if every attempt hits its cap. timeout-minutes: 45 strategy: matrix: shard: - auth-login - auth-register - auth-pages - research-workflow - research-form - research-metrics - settings-core - settings-pages - library - history-news - mobile - api-crud - error-benchmark - accessibility - chat-core - chat-lifecycle - link-analytics fail-fast: false permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar - name: Install Node.js for UI tests # zizmor: ignore[cache-poisoning] - caching explicitly disabled with cache: '' uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '24' # Cache poisoning protection: explicitly disable caching on PR events from forks cache: '' - name: Install UI test dependencies run: | export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true cd tests && npm ci # Combined "init DB ā†’ start server ā†’ register user ā†’ run tests" in a # retry block so each attempt gets fresh server + DB state, equivalent # to GitHub's "Re-run failed jobs" but skipping the docker artifact # re-download (the tarball is bit-identical between attempts). # # 3 attempts because 17 independent shards each at ~98% success only # gives ~75% gate pass; 2 retries brings per-shard fail to ~0.0001%. # More than 3 starts hiding genuine bugs (a test that fails 3/3 is a # regression, not a flake). - name: Run UI tests with retry (fresh server per attempt) uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 env: SHARD: ${{ matrix.shard }} TEST_IMAGE: ${{ env.TEST_IMAGE }} with: timeout_minutes: 12 max_attempts: 3 retry_on: error shell: bash # Runs after a failed attempt, before the next. Snapshots the # server log under a numbered filename so failure forensics # survive even when a later attempt passes. on_retry_command: | mkdir -p /tmp/ldr-ci-logs ATTEMPT=$(( $(find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${SHARD}-attempt-*.log" 2>/dev/null | wc -l) + 1 )) echo "::group::Cleanup after failed attempt ${ATTEMPT}" docker logs ldr-server > "/tmp/ldr-ci-logs/server-${SHARD}-attempt-${ATTEMPT}.log" 2>&1 || true docker stop ldr-server || true docker rm ldr-server || true sudo rm -rf "${GITHUB_WORKSPACE}/ci-data" || true echo "::endgroup::" command: | set -e mkdir -p "${GITHUB_WORKSPACE}/ci-data" echo "::group::Initialize test database" docker run --rm \ -v "${GITHUB_WORKSPACE}":/app \ -v "${GITHUB_WORKSPACE}/ci-data":/data \ -e TEST_ENV=true \ -e LDR_DATA_DIR=/data \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ -w /app/src \ "${TEST_IMAGE}" \ python ../scripts/ci/init_test_database.py echo "::endgroup::" # The link-analytics shard renders /metrics/links, whose domain # cards come entirely from ResearchResource rows. A fresh DB has # none, so seed a small fixture for this shard ONLY ā€” other shards # (and the empty-state screenshot suites) keep an empty DB. if [ "${SHARD}" = "link-analytics" ]; then echo "::group::Seed link analytics data" docker run --rm \ -v "${GITHUB_WORKSPACE}":/app \ -v "${GITHUB_WORKSPACE}/ci-data":/data \ -e TEST_ENV=true \ -e LDR_DATA_DIR=/data \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ -w /app/src \ "${TEST_IMAGE}" \ python ../scripts/ci/seed_link_analytics.py echo "::endgroup::" fi echo "::group::Start application server" docker run -d \ --name ldr-server \ -p 5000:5000 \ -v "${GITHUB_WORKSPACE}/ci-data":/data \ -e CI=true \ -e TEST_ENV=true \ -e FLASK_ENV=testing \ -e LDR_DISABLE_RATE_LIMITING=true \ -e SECRET_KEY=test-secret-key-for-ci \ -e LDR_DATA_DIR=/data \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ "${TEST_IMAGE}" ldr-web # --max-time bounds total request time; --connect-timeout bounds # TCP handshake. Without these, a hung connection would silently # eat the 12-min per-attempt budget instead of failing fast. for i in {1..60}; do if curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then echo "Server is ready after $i seconds" break fi if ! docker ps --filter "name=ldr-server" --filter "status=running" -q | grep -q .; then echo "Server container died!" echo "Server log:" docker logs ldr-server exit 1 fi echo "Waiting for server... ($i/60)" sleep 1 done if ! curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then echo "Server failed to start after 60 seconds" echo "Server log:" docker logs ldr-server exit 1 fi echo "::endgroup::" echo "::group::Register CI test user" docker run --rm \ -v "${GITHUB_WORKSPACE}":/app \ -e CI=true \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ --network host \ -w /app/tests/ui_tests \ "${TEST_IMAGE}" \ node register_ci_user.js http://localhost:5000 echo "::endgroup::" echo "::group::Run UI tests for shard ${SHARD}" docker run --rm \ -v "${GITHUB_WORKSPACE}":/app \ -e CI=true \ -e PUPPETEER_HEADLESS=true \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ --network host \ -w /app/tests/ui_tests \ "${TEST_IMAGE}" \ sh -c "xvfb-run -a -s '-screen 0 1920x1080x24' node run_all_tests.js --shard=${SHARD}" echo "::endgroup::" - name: Capture final server logs on failure if: failure() run: | mkdir -p /tmp/ldr-ci-logs ATTEMPT=$(( $(find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${{ matrix.shard }}-attempt-*.log" 2>/dev/null | wc -l) + 1 )) FINAL_LOG="/tmp/ldr-ci-logs/server-${{ matrix.shard }}-attempt-${ATTEMPT}-final.log" docker logs ldr-server > "$FINAL_LOG" 2>&1 || true { echo "## šŸŖµ ldr-server logs [${{ matrix.shard }}] ā€” final attempt (last 200 lines)" echo "" echo '```' tail -n 200 "$FINAL_LOG" || true echo '```' echo "" echo "Grepping for SQLAlchemy pool / DB session issues across all attempts:" echo '```' grep -hE "QueuePool|connection timed out|TimeoutError|cleanup_dead_threads|Failed to open database|DatabaseSessionError" /tmp/ldr-ci-logs/server-"${{ matrix.shard }}"-attempt-*.log 2>/dev/null | tail -n 50 || true echo '```' echo "" echo "Attempt logs preserved for upload:" find /tmp/ldr-ci-logs -maxdepth 1 -name "server-${{ matrix.shard }}-attempt-*.log" -printf '%TY-%Tm-%Td %TH:%TM %s\t%p\n' 2>/dev/null || true } >> "$GITHUB_STEP_SUMMARY" - name: Stop server if: always() run: | docker stop ldr-server || true docker rm ldr-server || true - name: Upload UI test screenshots if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ui-test-screenshots-${{ matrix.shard }} path: | tests/screenshots/ tests/ui_tests/screenshots/ - name: Upload server logs on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ui-tests-server-log-${{ matrix.shard }} path: /tmp/ldr-ci-logs/server-${{ matrix.shard }}-attempt-*.log retention-days: 3 if-no-files-found: ignore # Aggregates matrix-cell results into a single stable check name. # ci-gate.yml and any branch-protection rule should depend on this job's # name ("UI Tests (Puppeteer)"), not on individual shard cells. ui-tests-summary: runs-on: ubuntu-latest name: UI Tests (Puppeteer) needs: ui-tests if: ${{ always() && inputs.strict-mode == true }} permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - name: Check all shards succeeded run: | result="${{ needs.ui-tests.result }}" echo "Aggregated matrix result: $result" if [ "$result" != "success" ]; then echo "::error::One or more UI test shards did not succeed (result=$result)." exit 1 fi echo "All UI test shards passed." # Run accessibility tests (Playwright a11y + Lighthouse) accessibility-tests: runs-on: ubuntu-latest timeout-minutes: 15 name: Accessibility Tests needs: [build-test-image, detect-changes] if: needs.detect-changes.outputs.accessibility == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar - name: Initialize test database run: | mkdir -p "$PWD/ci-data" docker run --rm \ -v "$PWD":/app \ -v "$PWD/ci-data":/data \ -e TEST_ENV=true \ -e LDR_DATA_DIR=/data \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ -w /app/src \ ${{ env.TEST_IMAGE }} \ python ../scripts/ci/init_test_database.py - name: Start application server in Docker run: | docker run -d \ --name ldr-a11y-server \ -p 5000:5000 \ -v "$PWD/ci-data":/data \ -e CI=true \ -e TEST_ENV=true \ -e FLASK_ENV=testing \ -e LDR_DISABLE_RATE_LIMITING=true \ -e SECRET_KEY=test-secret-key-for-ci \ -e LDR_DATA_DIR=/data \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ ${{ env.TEST_IMAGE }} ldr-web # Wait for server to be ready. # --connect-timeout/--max-time bound TCP and total request time so a # hung connection fails fast instead of eating the job timeout. for i in {1..60}; do if curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then echo "Server is ready after $i seconds" break fi if ! docker ps --filter "name=ldr-a11y-server" --filter "status=running" -q | grep -q .; then echo "Server container died!" echo "Server log:" docker logs ldr-a11y-server exit 1 fi echo "Waiting for server... ($i/60)" sleep 1 done # Final check if ! curl -fsS --connect-timeout 2 --max-time 5 http://localhost:5000/api/v1/health 2>/dev/null; then echo "Server failed to start after 60 seconds" echo "Server log:" docker logs ldr-a11y-server exit 1 fi - name: Register CI test user run: | docker run --rm \ -v "$PWD":/app \ -e CI=true \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ -e NODE_PATH=/install/tests/ui_tests/node_modules \ --network host \ -w /app/tests/ui_tests \ ${{ env.TEST_IMAGE }} \ node register_ci_user.js http://localhost:5000 - name: Run Playwright accessibility tests timeout-minutes: 5 run: | docker run --rm \ -v "$PWD":/app \ -e CI=true \ -e BASE_URL=http://localhost:5000 \ -e LDR_DB_CONFIG_KDF_ITERATIONS=1000 \ -e LDR_TEST_MODE=1 \ --network host \ -w /app \ ${{ env.TEST_IMAGE }} \ bash -c '\ ln -sfn /install/tests/accessibility_tests/node_modules /app/node_modules && \ ln -sfn /install/tests/accessibility_tests/node_modules /app/tests/accessibility_tests/node_modules && \ /install/tests/accessibility_tests/node_modules/.bin/playwright test \ --config=/app/playwright.config.js --project=chromium; \ EXIT=$?; \ rm -f /app/node_modules /app/tests/accessibility_tests/node_modules; \ exit $EXIT' - name: Run Lighthouse accessibility audit if: always() timeout-minutes: 5 run: | docker run --rm \ -v "$PWD":/app \ -e CI=true \ --network host \ -w /app \ ${{ env.TEST_IMAGE }} \ bash -c '\ if [ ! -x /usr/local/bin/chrome ]; then echo "ERROR: Chrome not found at /usr/local/bin/chrome" >&2; exit 1; fi && \ echo "#!/bin/bash" > /tmp/chrome-wrapper.sh && \ echo "exec /usr/local/bin/chrome --no-sandbox --headless \"\$@\"" >> /tmp/chrome-wrapper.sh && \ chmod +x /tmp/chrome-wrapper.sh && \ CHROME_PATH=/tmp/chrome-wrapper.sh \ /install/tests/accessibility_tests/node_modules/.bin/lhci autorun --config=/app/lighthouserc.json' - name: Capture server logs on failure if: failure() run: | mkdir -p /tmp/ldr-ci-logs LOG_FILE=/tmp/ldr-ci-logs/ldr-a11y-server.log docker logs ldr-a11y-server > "$LOG_FILE" 2>&1 || true { echo "## šŸŖµ ldr-a11y-server logs (last 200 lines)" echo "" echo '```' tail -n 200 "$LOG_FILE" || true echo '```' } >> "$GITHUB_STEP_SUMMARY" - name: Upload accessibility reports if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: accessibility-reports path: | tests/accessibility_tests/playwright-report/ tests/accessibility_tests/test-results/ tests/accessibility_tests/.lighthouseci/ - name: Upload server logs on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: a11y-server-log path: /tmp/ldr-ci-logs/ldr-a11y-server.log retention-days: 3 if-no-files-found: ignore - name: Stop server if: always() run: | docker stop ldr-a11y-server || true docker rm ldr-a11y-server || true # Run LLM unit tests (conditional on path changes) llm-unit-tests: runs-on: ubuntu-latest name: LLM Unit Tests needs: [build-test-image, detect-changes] if: needs.detect-changes.outputs.llm == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar - name: Run LLM registry tests run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/test_llm/test_llm_registry.py -v -n auto" - name: Run LLM integration tests run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/test_llm/test_llm_integration.py -v -n auto" - name: Run API LLM integration tests run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/test_llm/test_api_llm_integration.py -v -n auto" - name: Run LLM edge case tests run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/test_llm/test_llm_edge_cases.py -v -n auto" - name: Run LLM benchmark tests run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "python -m pytest tests/test_llm/test_llm_benchmarks.py -v -n auto" # Run LLM example tests (conditional on path changes) llm-example-tests: runs-on: ubuntu-latest name: LLM Example Tests needs: [build-test-image, detect-changes] if: needs.detect-changes.outputs.llm == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true timeout-minutes: 20 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar # basic_custom_llm.py and advanced_custom_llm.py are intentionally NOT # executed in CI: they run real research pipelines against Wikipedia and # PubMed, which makes them flaky under a 60s budget. The examples are # kept in the repo as documentation and are covered by the compile check # below; mock_llm_example.py exercises the same integration path offline. - name: Test mock LLM example run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "timeout 60s python examples/llm_integration/mock_llm_example.py" - name: Test direct import run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "timeout 60s python examples/api_usage/programmatic/test_direct_import.py" - name: API public contract guardrail run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "timeout 60s python examples/api_usage/programmatic/api_public_contract_guardrail.py" - name: Compile check all remaining examples run: | docker run --rm \ -v "$PWD":/app \ -e PYTHONPATH=/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c " python -m py_compile examples/api_usage/programmatic/simple_programmatic_example.py && python -m py_compile examples/api_usage/programmatic/advanced_features_example.py && python -m py_compile examples/api_usage/programmatic/search_strategies_example.py && python -m py_compile examples/api_usage/programmatic/minimal_working_example.py && python -m py_compile examples/api_usage/programmatic/searxng_example.py && python -m py_compile examples/api_usage/programmatic/custom_llm_retriever_example.py && python -m py_compile examples/api_usage/programmatic/hybrid_search_example.py && python -m py_compile examples/llm_integration/basic_custom_llm.py && python -m py_compile examples/llm_integration/advanced_custom_llm.py && python -m py_compile examples/api_usage/http/simple_working_example.py && python -m py_compile examples/api_usage/http/advanced/http_api_examples.py && python -m py_compile examples/api_usage/http/advanced/simple_http_example.py && python -m py_compile examples/api_usage/simple_client_example.py && python -m py_compile examples/example_browsecomp.py && python -m py_compile examples/show_env_vars.py && python -m py_compile examples/run_benchmark.py && python -m py_compile examples/elasticsearch/search_example.py && echo 'All compile checks passed' " # Smoke test for the production ldr image ldr-production-smoke-test: runs-on: ubuntu-latest name: Production Image Smoke Test needs: detect-changes if: needs.detect-changes.outputs.docker == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true timeout-minutes: 30 permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build production ldr image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 with: context: . target: ldr load: true tags: ldr-prod # Cache poisoning protection: only read/write cache on trusted events (not PRs from forks) cache-from: ${{ github.event_name != 'pull_request' && 'type=gha,scope=ldr-prod' || '' }} cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=ldr-prod' || '' }} - name: Verify file ownership and playwright access run: | docker run --rm ldr-prod sh -c ' echo "Checking /install/.venv ownership..." VENV_OWNER=$(stat -c "%U:%G" /install/.venv) if [ "$VENV_OWNER" != "ldruser:ldruser" ]; then echo "FAIL: /install/.venv owned by $VENV_OWNER, expected ldruser:ldruser" exit 1 fi echo "OK: /install/.venv owned by $VENV_OWNER" echo "Spot-checking venv file ownership..." BAD_FILES=$(find /install/.venv \( ! -user ldruser -o ! -group ldruser \) -print -quit) if [ -n "$BAD_FILES" ]; then echo "FAIL: files not owned by ldruser:ldruser:" find /install/.venv \( ! -user ldruser -o ! -group ldruser \) | head -10 exit 1 fi echo "OK: venv files owned by ldruser:ldruser" echo "Checking /install parent dir ownership..." INSTALL_OWNER=$(stat -c "%U:%G" /install) if [ "$INSTALL_OWNER" != "ldruser:ldruser" ]; then echo "FAIL: /install owned by $INSTALL_OWNER, expected ldruser:ldruser" exit 1 fi echo "OK: /install owned by $INSTALL_OWNER" echo "Checking /scripts parent dir ownership..." SCRIPTS_OWNER=$(stat -c "%U:%G" /scripts) if [ "$SCRIPTS_OWNER" != "ldruser:ldruser" ]; then echo "FAIL: /scripts owned by $SCRIPTS_OWNER, expected ldruser:ldruser" exit 1 fi echo "OK: /scripts owned by $SCRIPTS_OWNER" echo "Checking /home/ldruser ownership..." HOME_OWNER=$(stat -c "%U:%G" /home/ldruser) if [ "$HOME_OWNER" != "ldruser:ldruser" ]; then echo "FAIL: /home/ldruser owned by $HOME_OWNER, expected ldruser:ldruser" exit 1 fi echo "OK: /home/ldruser owned by $HOME_OWNER" echo "Checking playwright is accessible as ldruser..." playwright --version echo "OK: playwright accessible as ldruser" echo "Checking playwright browsers are NOT in production image..." if [ -d /home/ldruser/.cache/ms-playwright ]; then echo "FAIL: Playwright browsers found in production image (should only be in ldr-test stage)" exit 1 fi echo "OK: no playwright browsers in production image (as expected)" ' - name: Start production container run: | mkdir -p "$PWD/ci-data-prod" # Run with the same security settings as docker-compose.yml so CI # catches capability regressions (e.g., missing cap_add that breaks # the entrypoint's chown/setpriv in restricted environments). docker run -d \ --name ldr-prod-test \ -p 5001:5000 \ -v "$PWD/ci-data-prod":/data \ --security-opt no-new-privileges:true \ --cap-drop ALL \ --cap-add CHOWN \ --cap-add FOWNER \ --cap-add DAC_OVERRIDE \ --cap-add SETUID \ --cap-add SETGID \ -e LDR_DATA_DIR=/data \ -e LDR_DISABLE_RATE_LIMITING=true \ -e SECRET_KEY=test-secret-key-for-ci \ ldr-prod - name: Wait for healthy status run: | echo "Waiting for server to be healthy..." for i in {1..60}; do if curl -sf --connect-timeout 2 --max-time 5 http://localhost:5001/api/v1/health > /dev/null 2>&1; then echo "Server is healthy after $i seconds" exit 0 fi if ! docker ps --filter "name=ldr-prod-test" --filter "status=running" -q | grep -q .; then echo "Container died!" docker logs ldr-prod-test exit 1 fi echo "Waiting... ($i/60)" sleep 1 done echo "Server failed to become healthy" docker logs ldr-prod-test exit 1 - name: Stop container if: always() run: | docker stop ldr-prod-test || true docker rm ldr-prod-test || true # Run infrastructure tests (conditional on path changes) infrastructure-tests: runs-on: ubuntu-latest name: Infrastructure Tests needs: [build-test-image, detect-changes] if: needs.detect-changes.outputs.infrastructure == 'true' || github.event_name == 'workflow_dispatch' || inputs.strict-mode == true permissions: contents: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download Docker image uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: docker-image path: /tmp - name: Load Docker image run: docker load --input /tmp/ldr-test.tar - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '24' - name: Install JavaScript test dependencies run: | cd tests/infrastructure_tests && npm ci - name: Run Python infrastructure tests run: | docker run --rm \ -v "$PWD":/app \ -w /app \ ${{ env.TEST_IMAGE }} \ sh -c "cd /app && python -m pytest tests/infrastructure_tests/test_*.py -v --color=yes --no-header -rN" - name: Run JavaScript infrastructure tests run: | cd tests/infrastructure_tests && npm test - name: Run Vitest JS unit tests run: | npm ci && npm test