tracer-cloud--opensre/.github/workflows/trigger-k8s-alert.yml

name: Trigger K8s Alert

on:
  workflow_dispatch:
    inputs:
      verify:
        description: "Verify logs in Datadog + post to Slack"
        type: boolean
        default: true
  schedule:
    - cron: "0 9 * * 1"  # every Monday 9am UTC

permissions:
  contents: read

jobs:
  trigger-alert:
    if: github.repository == 'Tracer-Cloud/opensre'
    runs-on: ubuntu-latest
    env:
      AWS_REGION: us-east-1
      DD_API_KEY: ${{ secrets.DD_API_KEY }}
      DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
      DD_SITE: datadoghq.com
      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
      SLACK_DEVS_ALERTS_CHANNEL_ID: C09S7GDG60J

    steps:
      - uses: actions/checkout@v5

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Install uv
        uses: astral-sh/setup-uv@v8.1.0
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Install project
        run: uv sync --frozen

      - name: Trigger pipeline failure via centralized config (~10s)
        id: trigger
        run: |
          echo "trigger_epoch=$(date +%s)" >> "$GITHUB_OUTPUT"
          set +e
          uv run python -m tests.e2e.kubernetes.trigger_alert | tee /tmp/trigger.log
          trigger_exit=${PIPESTATUS[0]}
          set -e
          echo "trigger_completed_epoch=$(date +%s)" >> "$GITHUB_OUTPUT"
          if grep -q '"status": "accepted_timeout"' /tmp/trigger.log; then
            echo "trigger_accepted_timeout=true" >> "$GITHUB_OUTPUT"
          else
            echo "trigger_accepted_timeout=false" >> "$GITHUB_OUTPUT"
          fi
          exit "$trigger_exit"

      - name: Report trigger result
        run: |
          echo "Trigger command completed successfully"

      - name: Verify Datadog + Slack (~5 min)
        if: ${{ inputs.verify == true || github.event_name == 'schedule' }}
        run: |
          POST_TRIGGER_WAIT=""
          if [ "${{ steps.trigger.outputs.trigger_accepted_timeout }}" = "true" ]; then
            POST_TRIGGER_WAIT="--post-trigger-wait 90"
          fi
          uv run python -m tests.e2e.kubernetes.trigger_alert \
            --verify-only \
            --since-epoch ${{ steps.trigger.outputs.trigger_completed_epoch }} \
            --dd-max-wait 300 \
            $POST_TRIGGER_WAIT