learningcircuit--local-deep-research/.github/workflows/nuclei.yml

name: Nuclei Vulnerability Scan

# Template-driven DAST scanner — complements ZAP with known-CVE checks,
# misconfiguration detection, exposed panel discovery, and default credential
# testing. Uses ProjectDiscovery's 6,500+ community-maintained templates.
#
# Authenticated scan: the runner pre-creates the standard CI test_admin user,
# logs in via the real /auth/login flow to capture a session cookie, and
# seeds Nuclei with the full Flask url_map so the scanner probes the entire
# authenticated app surface (settings, research, history, API, …) — not just
# the unauthenticated landing page.

on:
  workflow_call:  # Called by release-gate.yml
  workflow_dispatch:

permissions: {}

jobs:
  nuclei-scan:
    name: Nuclei DAST Scan
    runs-on: ubuntu-latest
    timeout-minutes: 40
    permissions:
      contents: read
      security-events: write
      actions: read

    env:
      CI: true
      TEST_ENV: true
      FLASK_ENV: testing
      LDR_DATA_DIR: ${{ github.workspace }}/data
      LDR_DB_CONFIG_KDF_ITERATIONS: "1000"
      # Honour the fast test KDF: without LDR_TEST_MODE the value is clamped
      # to the production minimum (256000), making every SQLCipher open/backup
      # ~256x slower — see #4430 and tests/shared/chrome_profile.js context.
      LDR_TEST_MODE: "1"
      LDR_DISABLE_RATE_LIMITING: "true"
      SECRET_KEY: test-secret-key-for-ci
      TEST_USERNAME: test_admin
      TEST_PASSWORD: testpass123

    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
        with:
          egress-policy: audit

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.12'

      - name: Set up PDM
        uses: pdm-project/setup-pdm@973541a5febeafcfdadf8a51211435be6ecfd90f # v4.5
        with:
          python-version: '3.12'

      - name: Install system dependencies for SQLCipher
        run: |
          sudo apt-get update
          sudo apt-get install -y libsqlcipher-dev

      - name: Install dependencies
        run: |
          pdm sync -d

      - name: Setup test data directory
        run: |
          mkdir -p "$LDR_DATA_DIR/encrypted_databases"

      - name: Pre-create CI test user
        # Avoids the slow registration path (KDF + 500 settings rows) and
        # avoids hitting the registration rate limit during the scan window.
        run: |
          export PYTHONPATH=$PWD/src:$PYTHONPATH
          pdm run python scripts/ci/init_test_database.py

      - name: Start LDR server for testing
        run: |
          export PYTHONPATH=$PWD/src:$PYTHONPATH
          pdm run python -m local_deep_research.web.app > server.log 2>&1 &
          SERVER_PID=$!
          echo "$SERVER_PID" > server.pid

          # Wait for server to start.
          # --connect-timeout/--max-time bound TCP and total request time so a
          # hung connection fails fast instead of eating the job timeout.
          for _ in {1..90}; do
            if curl -fsS --connect-timeout 2 --max-time 5 http://127.0.0.1:5000/api/v1/health > /dev/null; then
              echo "Server started successfully"
              break
            fi
            sleep 1
          done

          # Verify server is running
          if ! curl -fsS --connect-timeout 2 --max-time 5 http://127.0.0.1:5000/api/v1/health > /dev/null; then
            echo "Server failed to start"
            cat server.log
            exit 1
          fi

      - name: Dump Flask url_map for URL seeding
        # Without -list, Nuclei only probes the single -target URL. Seeding
        # with the real url_map lets it exercise every blueprint route.
        run: |
          export PYTHONPATH=$PWD/src:$PYTHONPATH
          pdm run python scripts/ci/dump_url_map.py http://127.0.0.1:5000 > urls.txt
          echo "Seeded $(wc -l < urls.txt) URLs:"
          head -20 urls.txt
          echo "..."

      - name: Authenticate and capture session cookie
        id: login
        run: |
          # Step 1: GET /auth/login → establishes a Flask session cookie and
          # returns the HTML form with the per-session CSRF token.
          curl -sS -c cookies.txt -o login.html http://127.0.0.1:5000/auth/login

          CSRF=$(grep -oE 'name="csrf_token"[[:space:]]+value="[^"]+"' login.html \
            | head -n1 \
            | sed -E 's/.*value="([^"]+)".*/\1/')
          if [ -z "$CSRF" ]; then
            echo "Failed to extract CSRF token from login page"
            head -200 login.html
            exit 1
          fi

          # Step 2: POST credentials with the CSRF token and the cookie jar.
          # -L follows the post-login 302 to /. -o /dev/null discards body,
          # -w prints the final HTTP code so a non-2xx fails the step.
          HTTP_CODE=$(curl -sS -L \
            -b cookies.txt -c cookies.txt \
            -o /dev/null -w '%{http_code}' \
            --data-urlencode "username=${TEST_USERNAME}" \
            --data-urlencode "password=${TEST_PASSWORD}" \
            --data-urlencode "csrf_token=${CSRF}" \
            http://127.0.0.1:5000/auth/login)
          if [ "$HTTP_CODE" != "200" ]; then
            echo "Login failed with HTTP $HTTP_CODE"
            cat server.log
            exit 1
          fi

          # Verify the session is actually authenticated (not just that
          # the login page rendered with a 200 from a re-display of errors).
          AUTHED=$(curl -sS -b cookies.txt http://127.0.0.1:5000/auth/check)
          echo "auth/check response: $AUTHED"
          echo "$AUTHED" | grep -q '"authenticated":[[:space:]]*true' || {
            echo "Authenticated check failed"
            exit 1
          }

          # Extract the session cookie value from the Netscape cookie jar.
          # Format: domain TAB tailmatch TAB path TAB secure TAB expires TAB name TAB value
          SESSION=$(awk '$6 == "session" { print $7 }' cookies.txt | tail -n1)
          if [ -z "$SESSION" ]; then
            # Print every column EXCEPT the value ($7) so we can debug
            # without leaking the cookie if the awk filter is broken.
            echo "Could not find 'session' cookie in jar (values redacted):"
            awk '{ print $1, $2, $3, $4, $5, $6 }' cookies.txt
            exit 1
          fi
          # Mask the cookie in logs — it grants full app access for this run.
          echo "::add-mask::$SESSION"
          echo "session=$SESSION" >> "$GITHUB_OUTPUT"

          # Give the post-login background thread a moment to finish
          # the settings-migration / library-init pass it kicks off
          # (see _perform_post_login_tasks in web/auth/routes.py).
          # Otherwise the first authenticated probes can race those
          # writes and 500 on settings-dependent routes.
          sleep 2

      - name: Create empty SARIF fallback
        run: |
          cat > nuclei.sarif << 'SARIF'
          {
            "version": "2.1.0",
            "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
            "runs": [{
              "tool": {
                "driver": {
                  "name": "Nuclei",
                  "informationUri": "https://github.com/projectdiscovery/nuclei",
                  "rules": []
                }
              },
              "results": []
            }]
          }
          SARIF

      - name: Run Nuclei scan
        uses: projectdiscovery/nuclei-action@cc153d0541e1adf8a42bbe31c0a4fb2376147538 # v3.1.1
        with:
          # -list: probe every route in the Flask url_map (not just /).
          # -H: attach the authenticated session cookie so probes hit the
          #     real app surface, not the login redirect.
          # -severity: drop info-level noise (form-detection, options-method,
          #     intentional CSP/cookie choices). Real findings are >= low.
          # -etags intrusive,dos,fuzz: with a live session, default templates
          #     can mutate state or DoS the runner. Exclude the standard
          #     destructive tag set for authenticated DAST.
          # -eid http-missing-security-headers: HSTS is correctly omitted on
          #     plain HTTP localhost; X-XSS-Protection is intentionally
          #     omitted (deprecated, replaced by CSP). See
          #     security/security_headers.py.
          args: >-
            -list urls.txt
            -H "Cookie: session=${{ steps.login.outputs.session }}"
            -severity low,medium,high,critical
            -etags intrusive,dos,fuzz
            -eid http-missing-security-headers
            -sarif-export nuclei.sarif
            -output nuclei.log

      - name: Upload Nuclei SARIF to GitHub Security tab
        uses: github/codeql-action/upload-sarif@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
        if: always()
        with:
          sarif_file: nuclei.sarif
          category: nuclei-dast

      - name: Upload Nuclei scan artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: nuclei-scan-results
          path: |
            nuclei.log
            nuclei.sarif
            urls.txt
            server.log
          retention-days: 30

      - name: Stop server
        if: always()
        run: |
          if [ -f server.pid ]; then
            kill "$(cat server.pid)" || true
            rm server.pid
          fi