czlonkowski--n8n-mcp/.github/workflows/test.yml

name: Test Suite
on:
  push:
    branches: [main, feat/comprehensive-testing-suite]
    paths-ignore:
      - '**.md'
      - '**.txt'
      - 'docs/**'
      - 'examples/**'
      - '.github/FUNDING.yml'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/pull_request_template.md'
      - '.gitignore'
      - 'LICENSE*'
      - 'ATTRIBUTION.md'
      - 'SECURITY.md'
      - 'CODE_OF_CONDUCT.md'
  pull_request:
    branches: [main]
    paths-ignore:
      - '**.md'
      - '**.txt'
      - 'docs/**'
      - 'examples/**'
      - '.github/FUNDING.yml'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/pull_request_template.md'
      - '.gitignore'
      - 'LICENSE*'
      - 'ATTRIBUTION.md'
      - 'SECURITY.md'
      - 'CODE_OF_CONDUCT.md'

permissions:
  contents: read
  issues: write
  pull-requests: write
  checks: write

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 15  # Increased from 10 to accommodate larger database with community nodes
    env:
      # A boolean flag (not the secrets themselves) recording whether a live n8n
      # instance is configured, so the live-integration step can gate on it without
      # the N8N_API_* values being exposed to every step in the job. The live tests
      # need BOTH the URL and the key (getN8nCredentials() throws if either is
      # missing), and both are empty on Dependabot/fork PRs — so require both here,
      # otherwise a partially-configured instance would run the step and then fail.
      HAS_N8N_INSTANCE: ${{ secrets.N8N_API_URL != '' && secrets.N8N_API_KEY != '' }}
    steps:
      - uses: actions/checkout@v7
      
      - uses: actions/setup-node@v6
        with:
          node-version: 20
          cache: 'npm'
      
      - name: Install dependencies
        # --legacy-peer-deps: n8n-nodes-base 2.20.x pulls in mappersmith with a `diff`
        # peer that conflicts with ts-node's `diff@^4`; npm 7+ strict resolution leaves
        # the lock internally inconsistent. Match local dev (.npmrc legacy-peer-deps=true).
        run: npm ci --legacy-peer-deps
      
      # Verify test environment setup
      - name: Verify test environment
        run: |
          echo "Current directory: $(pwd)"
          echo "Checking for .env.test file:"
          ls -la .env.test || echo ".env.test not found!"
          echo "First few lines of .env.test:"
          head -5 .env.test || echo "Cannot read .env.test"
      
      # Run unit tests first (without MSW)
      - name: Run unit tests with coverage
        run: npm run test:unit -- --coverage --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0 --reporter=default --reporter=junit
        env:
          CI: true
      
      # Pre-build the Docker test image used by tests/integration/docker/*.test.ts
      # Two test files used to build it independently inside their beforeAll hooks; the
      # builder stage does an `npm install` from the public registry which intermittently
      # fails on CI runners and tanked the entire test job. Building it here once gets us:
      #  - one build attempt instead of two
      #  - a clear, separately-visible CI step when the npm registry is flaky
      #  - the test suite degrades to skipped tests if this step fails (won't block PRs)
      - name: Build Docker test image
        run: npm run docker:test:build
        timeout-minutes: 8
        continue-on-error: true

      # Offline integration suites (database, security, mcp-protocol, workflow-diff,
      # templates, docker, …). These need no live instance, so they always run and
      # stay a blocking gate on every PR — Dependabot and forks included. Only the
      # live n8n-api / ai-validation suites are excluded here (they run in the next
      # step). MSW setup applies as configured.
      - name: Run integration tests (offline suites)
        run: npm run test:integration -- --exclude 'tests/integration/n8n-api/**' --exclude 'tests/integration/ai-validation/**' --reporter=default --reporter=junit
        env:
          CI: true

      # Live n8n-API + AI-validation suites hit a real instance via the N8N_API_*
      # secrets, which are unavailable on Dependabot and fork PRs (there every file
      # here fails outright, turning the required `test` check permanently red). Skip
      # them when no instance is configured; on in-repo PRs the secret is present so
      # they run and remain a blocking gate. Secrets are scoped to this step only.
      - name: Run integration tests (live n8n instance)
        if: ${{ env.HAS_N8N_INSTANCE == 'true' }}
        run: npm run test:integration -- tests/integration/n8n-api tests/integration/ai-validation --reporter=default --reporter=junit
        env:
          CI: true
          N8N_API_URL: ${{ secrets.N8N_API_URL }}
          N8N_API_KEY: ${{ secrets.N8N_API_KEY }}
          N8N_TEST_WEBHOOK_GET_URL: ${{ secrets.N8N_TEST_WEBHOOK_GET_URL }}
          N8N_TEST_WEBHOOK_POST_URL: ${{ secrets.N8N_TEST_WEBHOOK_POST_URL }}
          N8N_TEST_WEBHOOK_PUT_URL: ${{ secrets.N8N_TEST_WEBHOOK_PUT_URL }}
          N8N_TEST_WEBHOOK_DELETE_URL: ${{ secrets.N8N_TEST_WEBHOOK_DELETE_URL }}
      
      # Generate test summary
      - name: Generate test summary
        if: always()
        run: node scripts/generate-test-summary.js
      
      # Generate detailed reports
      - name: Generate detailed reports
        if: always()
        run: node scripts/generate-detailed-reports.js
      
      # Upload test results artifacts
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: test-results-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            test-results/
            test-summary.md
            test-reports/
          retention-days: 30
          if-no-files-found: warn
      
      # Upload coverage artifacts
      - name: Upload coverage reports
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: coverage-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            coverage/
          retention-days: 30
          if-no-files-found: warn
      
      # Upload coverage to Codecov
      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage/lcov.info
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false
          verbose: true
      
      # Run linting
      - name: Run linting
        run: npm run lint
      
      # Run type checking
      - name: Run type checking
        run: npm run typecheck
      
      # Create test report comment for PRs
      - name: Create test report comment
        if: github.event_name == 'pull_request' && always()
        uses: actions/github-script@v7
        continue-on-error: true
        with:
          script: |
            const fs = require('fs');
            let summary = '## Test Results\n\nTest summary generation failed.';
            
            try {
              if (fs.existsSync('test-summary.md')) {
                summary = fs.readFileSync('test-summary.md', 'utf8');
              }
            } catch (error) {
              console.error('Error reading test summary:', error);
            }
            
            try {
              // Find existing comment
              const { data: comments } = await github.rest.issues.listComments({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
              });
              
              const botComment = comments.find(comment => 
                comment.user.type === 'Bot' && 
                comment.body.includes('## Test Results')
              );
              
              if (botComment) {
                // Update existing comment
                await github.rest.issues.updateComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  comment_id: botComment.id,
                  body: summary
                });
              } else {
                // Create new comment
                await github.rest.issues.createComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: context.issue.number,
                  body: summary
                });
              }
            } catch (error) {
              console.error('Failed to create/update PR comment:', error.message);
              console.log('This is likely due to insufficient permissions for external PRs.');
              console.log('Test results have been saved to the job summary instead.');
            }
      
      # Generate job summary
      - name: Generate job summary
        if: always()
        run: |
          echo "# Test Run Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          
          if [ -f test-summary.md ]; then
            cat test-summary.md >> $GITHUB_STEP_SUMMARY
          else
            echo "Test summary generation failed." >> $GITHUB_STEP_SUMMARY
          fi
          
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "## 📥 Download Artifacts" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- [Test Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
          echo "- [Coverage Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
      
      # Store test metadata
      - name: Store test metadata
        if: always()
        run: |
          cat > test-metadata.json << EOF
          {
            "run_id": "${{ github.run_id }}",
            "run_number": "${{ github.run_number }}",
            "run_attempt": "${{ github.run_attempt }}",
            "sha": "${{ github.sha }}",
            "ref": "${{ github.ref }}",
            "event_name": "${{ github.event_name }}",
            "repository": "${{ github.repository }}",
            "actor": "${{ github.actor }}",
            "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
            "node_version": "$(node --version)",
            "npm_version": "$(npm --version)"
          }
          EOF
      
      - name: Upload test metadata
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: test-metadata-${{ github.run_number }}-${{ github.run_attempt }}
          path: test-metadata.json
          retention-days: 30

  # Verify the compiled CommonJS artifact loads under Node's CJS loader (regression guard for #864).
  # Node 20.19+/22.12+ silently tolerate require() of ESM-only deps; scripts/smoke-cjs-runtime.js
  # forces the strict loader (--no-experimental-require-module) when the running Node supports it,
  # so a CJS/ESM mismatch in a shipped dependency (e.g. uuid@14) fails here regardless of the
  # runner's Node version. The unit suite runs under Vitest's ESM pipeline and cannot catch this;
  # tsc only type-checks.
  cjs-runtime:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v6
        with:
          node-version: 20
          cache: 'npm'

      - name: Install dependencies
        run: npm ci --legacy-peer-deps

      - name: Build
        run: npm run build

      - name: Load compiled CommonJS artifact under strict CJS loader
        run: npm run test:cjs-runtime

  # Publish test results as checks
  publish-results:
    needs: test
    runs-on: ubuntu-latest
    if: always()
    permissions:
      checks: write
    steps:
      - uses: actions/checkout@v7

      - name: Download test results
        uses: actions/download-artifact@v8
        with:
          path: artifacts

      - name: Publish test results
        uses: dorny/test-reporter@v3
        if: always()
        continue-on-error: true
        with:
          name: Test Results
          path: 'artifacts/test-results-*/test-results/junit.xml'
          reporter: java-junit
          fail-on-error: false
          fail-on-empty: false