comet-ml--opik/.github/workflows/e2e_tests_post_merge_v2.yml

name: E2E Tests - Post Merge (v2)
run-name: "E2E Tests v2 (${{ github.event.inputs.tier || 't1' }}) - ${{ github.event_name == 'push' && format('Post-merge {0}', github.sha) || format('Manual run by @{0}', github.actor) }}"

on:
  push:
    branches:
      - 'main'
    # Skip workflow if only documentation or unrelated files changed
    paths-ignore:
      - '**.md'
      - 'apps/opik-documentation/**'

  pull_request:
    paths:
      - '.github/workflows/e2e_tests_post_merge_v2.yml'

  workflow_dispatch:
    inputs:
      tier:
        type: choice
        description: 'Which tag tier to run (t1=smoke, t2=cuj, t3=nightly)'
        required: true
        default: 't1'
        options:
          - t1
          - t2
          - t3
      notify_on_success:
        type: boolean
        description: 'Send Slack notification even on success'
        required: false
        default: false
      ALLURE_JOB_RUN_ID:
        description: 'ALLURE_JOB_RUN_ID service parameter. Leave blank.'
        required: false

env:
  ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
  ALLURE_ENDPOINT: https://comet.testops.cloud/
  ALLURE_PROJECT_ID: 1
  ALLURE_RESULTS: allure-results
  ALLURE_JOB_RUN_ID: ${{ github.event.inputs.ALLURE_JOB_RUN_ID }}

  OPIK_SENTRY_ENABLE: false

permissions:
  contents: read
  actions: read

jobs:
  e2e-tests:
    name: "🧪 E2E v2 Tests (${{ github.event.inputs.tier || 't1' }})"
    runs-on: ubuntu-latest
    timeout-minutes: 45

    outputs:
      test_result: ${{ steps.run-tests.outcome }}
      testops_url: ${{ steps.set-outputs.outputs.testops_url }}
      total_tests: ${{ steps.parse-results.outputs.total }}
      passed_tests: ${{ steps.parse-results.outputs.passed }}
      failed_tests: ${{ steps.parse-results.outputs.failed }}
      skipped_tests: ${{ steps.parse-results.outputs.skipped }}
      pr_author: ${{ steps.get-pr-info.outputs.author }}

    steps:
      # ========================================
      # PHASE 1: Checkout & Environment Setup
      # ========================================
      - name: "📥 Checkout repository"
        uses: actions/checkout@v6

      - name: "🔍 Get PR/commit information"
        id: get-pr-info
        run: |
          if [ "${{ github.event_name }}" == "push" ]; then
            # For push events, get the author from the commit
            AUTHOR="${{ github.event.head_commit.author.username }}"
            if [ -z "$AUTHOR" ]; then
              AUTHOR="${{ github.actor }}"
            fi
          else
            AUTHOR="${{ github.actor }}"
          fi
          echo "author=$AUTHOR" >> "$GITHUB_OUTPUT"
          echo "📝 Author: $AUTHOR"

      - name: "📦 Setup Node.js"
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: tests_end_to_end/e2e/package-lock.json

      - name: "🐍 Setup Python"
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: "⚡ Setup uv"
        uses: astral-sh/setup-uv@v5

      - name: "📚 Install E2E test dependencies"
        working-directory: tests_end_to_end/e2e
        run: |
          npm ci
          npx playwright install --with-deps chromium

      - name: "📚 Sync bridge dependencies"
        working-directory: tests_end_to_end/e2e/services/opik-sdk-driver
        run: uv sync

      - name: "📊 Install allurectl"
        uses: allure-framework/setup-allurectl@v1

      # ========================================
      # PHASE 2: Build & Start Opik
      # ========================================
      - name: "🔨 Build and start Opik server"
        env:
          OPIK_USAGE_REPORT_ENABLED: false
          COMPOSE_BAKE: false
        run: |
          echo "::group::Building Opik from branch: ${{ github.ref_name }}"
          cd ${{ github.workspace }}
          TOGGLE_WELCOME_WIZARD_ENABLED="false" TOGGLE_FORCE_WORKSPACE_VERSION="version_2" ./opik.sh --build
          echo "::endgroup::"
          echo "✅ Opik build complete"

      - name: "🔍 Verify Docker pods are running"
        run: |
          echo "::group::Checking Docker pods"
          chmod +x ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh
          ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh
          echo "::endgroup::"
          echo "✅ All Docker pods are running"

      - name: "🏥 Verify backend health"
        run: |
          echo "::group::Checking backend health"
          chmod +x ./tests_end_to_end/installer_utils/check_backend.sh
          ./tests_end_to_end/installer_utils/check_backend.sh
          echo "::endgroup::"
          echo "✅ Backend is healthy"

      # ========================================
      # PHASE 3: Run E2E Tests
      # ========================================
      - name: "🧪 Run E2E test suite"
        id: run-tests
        working-directory: tests_end_to_end/e2e
        env:
          OPIK_DEPLOYMENT: oss
          OPIK_BASE_URL: http://localhost:5173
          OPIK_WORKSPACE: default
          WORKERS: 2
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ALLURE_LAUNCH_NAME: "Opik v2 Post-Merge ${{ github.event.inputs.tier || 't1' }} - ${{ github.run_id }}"
        run: |
          TIER="${{ github.event.inputs.tier || 't1' }}"
          echo "========================================"
          echo "🧪 Running v2 tier: $TIER"
          echo "========================================"
          echo ""

          # Point both the allure-playwright reporter and allurectl at the same
          # absolute results dir, then stream results to TestOps via allurectl watch.
          export ALLURE_RESULTS="${{ github.workspace }}/tests_end_to_end/e2e/${{ env.ALLURE_RESULTS }}"

          # Create .allure directory to prevent testplan.json error
          mkdir -p .allure

          # Run tests with allurectl watch for TestOps integration
          # Capture output to extract the direct report URL
          set +e
          allurectl watch -- npm run test:${TIER} 2>&1 | tee allure_output.log
          TEST_EXIT_CODE=${PIPESTATUS[0]}
          set -e

          # Extract the direct job run URL from allurectl output
          if grep -q "Report link:" allure_output.log; then
            REPORT_URL=$(grep "Report link:" allure_output.log | head -1 | sed 's/.*Report link:[[:space:]]*//' | awk '{print $1}')
            echo "allure_report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT"
            echo "📊 Captured Allure TestOps report URL: ${REPORT_URL}"
          else
            echo "allure_report_url=" >> "$GITHUB_OUTPUT"
            echo "⚠️ Could not capture direct report URL from allurectl output"
          fi

          echo ""
          echo "========================================"
          echo "Test run completed with exit code: $TEST_EXIT_CODE"
          echo "========================================"

          exit "$TEST_EXIT_CODE"

      # ========================================
      # PHASE 4: Process Results
      # ========================================
      - name: "📊 Parse test results"
        id: parse-results
        if: always()
        working-directory: tests_end_to_end/e2e
        run: |
          echo "::group::Parsing test results"

          if [ -f "test-results/results.json" ]; then
            echo "Found results.json, parsing..."
            TOTAL=$(jq '.stats.expected + .stats.unexpected + .stats.flaky + .stats.skipped' test-results/results.json 2>/dev/null || echo "0")
            PASSED=$(jq '.stats.expected' test-results/results.json 2>/dev/null || echo "0")
            FAILED=$(jq '.stats.unexpected' test-results/results.json 2>/dev/null || echo "0")
            SKIPPED=$(jq '.stats.skipped' test-results/results.json 2>/dev/null || echo "0")
            FLAKY=$(jq '.stats.flaky' test-results/results.json 2>/dev/null || echo "0")

            echo "Results: Total=$TOTAL, Passed=$PASSED, Failed=$FAILED, Skipped=$SKIPPED, Flaky=$FLAKY"
          else
            echo "⚠️ results.json not found, using defaults"
            TOTAL="0"
            PASSED="0"
            FAILED="0"
            SKIPPED="0"
            FLAKY="0"
          fi

          {
            echo "total=$TOTAL"
            echo "passed=$PASSED"
            echo "failed=$FAILED"
            echo "skipped=$SKIPPED"
            echo "flaky=$FLAKY"
          } >> "$GITHUB_OUTPUT"

          echo "::endgroup::"

      - name: "🔗 Set output URLs"
        id: set-outputs
        if: always()
        run: |
          # Use direct report URL if captured, otherwise fall back to launches page
          DIRECT_URL="${{ steps.run-tests.outputs.allure_report_url }}"

          if [ -n "$DIRECT_URL" ]; then
            TESTOPS_URL="$DIRECT_URL"
            echo "📊 Using direct TestOps report URL"
          else
            TESTOPS_URL="${{ env.ALLURE_ENDPOINT }}project/${{ env.ALLURE_PROJECT_ID }}/launches"
            echo "📊 Using fallback launches page URL"
          fi

          echo "testops_url=$TESTOPS_URL" >> "$GITHUB_OUTPUT"

      # ========================================
      # PHASE 5: Generate Summary
      # ========================================
      - name: "📋 Generate job summary"
        if: always()
        run: |
          TIER="${{ github.event.inputs.tier || 't1' }}"
          TESTOPS_URL="${{ steps.set-outputs.outputs.testops_url }}"

          TOTAL="${{ steps.parse-results.outputs.total }}"
          PASSED="${{ steps.parse-results.outputs.passed }}"
          FAILED="${{ steps.parse-results.outputs.failed }}"
          SKIPPED="${{ steps.parse-results.outputs.skipped }}"

          # Redirect stdout to the summary file for the remainder of this step
          exec >> "$GITHUB_STEP_SUMMARY"

          # Header
          echo "# 🧪 E2E v2 Test Results"
          echo ""

          # Status banner
          if [ "${{ steps.run-tests.outcome }}" == "success" ]; then
            echo "> ✅ **All tests passed!**"
          elif [ "${{ steps.run-tests.outcome }}" == "failure" ]; then
            echo "> ❌ **$FAILED test(s) failed** - Please review the results below"
          else
            echo "> ⚠️ **Test run status: ${{ steps.run-tests.outcome }}**"
          fi
          echo ""

          # Configuration table
          echo "## 📝 Configuration"
          echo ""
          echo "| Property | Value |"
          echo "|----------|-------|"
          echo "| **Tier** | \`$TIER\` |"
          echo "| **Branch** | \`${{ github.ref_name }}\` |"
          echo "| **Commit** | [\`${GITHUB_SHA:0:7}\`](https://github.com/${{ github.repository }}/commit/${{ github.sha }}) |"
          echo "| **Trigger** | ${{ github.event_name == 'push' && 'Post-merge (push to main)' || 'Manual dispatch' }} |"
          echo "| **Author** | @${{ steps.get-pr-info.outputs.author }} |"
          echo ""

          # Results table
          echo "## 📊 Results"
          echo ""
          echo "| Status | Count |"
          echo "|--------|------:|"
          echo "| ✅ Passed | $PASSED |"
          echo "| ❌ Failed | $FAILED |"
          echo "| ⏭️ Skipped | $SKIPPED |"
          echo "| **Total** | **$TOTAL** |"
          echo ""

          # Links section
          echo "## 🔗 Quick Links"
          echo ""
          echo "| Resource | Link |"
          echo "|----------|------|"
          echo "| 📊 **TestOps Dashboard** | [View detailed results, traces & history]($TESTOPS_URL) |"
          echo "| 📄 **Playwright Report** | [Download from Artifacts below](#artifacts) |"
          echo ""

          # Debugging section (only on failure)
          if [ "${{ steps.run-tests.outcome }}" == "failure" ]; then
            echo "## 🔧 Debugging Guide"
            echo ""
            echo "### Quick steps to investigate failures:"
            echo ""
            echo "1. **Open TestOps** → Click the link above to see detailed failure traces, screenshots, and video recordings"
            echo "2. **Download Playwright Report** → The HTML report includes step-by-step traces for each test"
            echo "3. **Check test logs** → Expand the 'Run E2E test suite' step above for console output"
            echo ""
            echo "### Common failure causes:"
            echo "- UI element selectors changed (missing data-testid)"
            echo "- API response format changed"
            echo "- Timing/race conditions"
            echo "- New feature requiring test updates"
          fi

      # ========================================
      # PHASE 6: Upload Artifacts
      # ========================================
      - name: "📤 Upload Allure results"
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: allure-results-v2
          path: tests_end_to_end/e2e/allure-results
          retention-days: 14

      - name: "📤 Upload Playwright report"
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: playwright-report-v2
          path: tests_end_to_end/e2e/playwright-report
          retention-days: 14

      - name: "📤 Upload test results JSON"
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: test-results-json-v2
          path: tests_end_to_end/e2e/test-results/results.json
          retention-days: 14
          if-no-files-found: ignore

      - name: "🧹 Stop Opik server"
        if: always()
        run: |
          echo "Stopping Opik server..."
          ./opik.sh --stop || true
          echo "✅ Cleanup complete"

  notify-slack:
    name: "📢 Slack Notification"
    runs-on: ubuntu-latest
    needs: e2e-tests
    if: |
      always() &&
      needs.e2e-tests.result != 'cancelled' &&
      (needs.e2e-tests.result == 'failure' || github.event.inputs.notify_on_success == 'true')

    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          sparse-checkout: .github/scripts
          sparse-checkout-cone-mode: false

      - name: Resolve Slack mentions
        id: resolve-slack-mentions
        env:
          SLACK_USER_MAPPING: ${{ secrets.SLACK_USER_MAPPING }}
          SLACK_QA_LEAD_ID: ${{ secrets.SLACK_QA_LEAD_ID }}
        run: |
          AUTHOR="${{ needs.e2e-tests.outputs.pr_author }}"
          IS_FAILURE="${{ needs.e2e-tests.result == 'failure' }}"

          if [ "$IS_FAILURE" == "true" ]; then
            if [ -n "$SLACK_QA_LEAD_ID" ]; then
              QA_MENTION="<@$SLACK_QA_LEAD_ID>"
            else
              QA_MENTION=""
            fi

            AUTHOR_MENTION=""
            if [ -n "$SLACK_USER_MAPPING" ]; then
              AUTHOR_SLACK_ID=$(echo "$SLACK_USER_MAPPING" | jq -r --arg user "$AUTHOR" '.[$user] // empty' 2>/dev/null || echo "")
              if [ -n "$AUTHOR_SLACK_ID" ] && [ "$AUTHOR_SLACK_ID" != "$SLACK_QA_LEAD_ID" ]; then
                AUTHOR_MENTION="<@$AUTHOR_SLACK_ID>"
              fi
            fi

            MENTIONS="${QA_MENTION:+$QA_MENTION }${AUTHOR_MENTION}"
            echo "mentions=${MENTIONS% }" >> "$GITHUB_OUTPUT"
          else
            echo "mentions=" >> "$GITHUB_OUTPUT"
          fi

          echo "author_display=$AUTHOR" >> "$GITHUB_OUTPUT"

      - name: Send Slack notification
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          TEST_RESULT: ${{ needs.e2e-tests.result }}
          SUITE_NAME: "v2 ${{ github.event.inputs.tier || 't1' }}"
          PASSED_TESTS: ${{ needs.e2e-tests.outputs.passed_tests }}
          FAILED_TESTS: ${{ needs.e2e-tests.outputs.failed_tests }}
          TESTOPS_URL: ${{ needs.e2e-tests.outputs.testops_url }}
          AUTHOR_DISPLAY: ${{ steps.resolve-slack-mentions.outputs.author_display }}
          MENTIONS: ${{ steps.resolve-slack-mentions.outputs.mentions }}
        run: .github/scripts/notify-slack-e2e-post-merge.sh