tracer-cloud--opensre/.github/workflows/benchmark-run.yml

name: Benchmark — run on Fargate (any adapter)

# Manually-triggered benchmark run on AWS Fargate.
#
# Adapter-agnostic. This workflow does NOT know or care which benchmark
# is running. The ``config`` input names a YAML file; the YAML names an
# adapter (``benchmark: <name>``); the framework's registry resolves it
# to a registered ``BenchmarkAdapter`` subclass. The same workflow runs
# any adapter packaged into the bench image — CloudOpsBench today,
# ToolCallBench / future adapters tomorrow, with zero changes
# here. See ``tests/benchmarks/_framework/registry.py``.
#
# Why ECS RunTask instead of a GitHub-hosted ubuntu runner: a full bench
# grid runs for hours and writes hundreds of MB of artifacts — Fargate
# handles it cleanly, has AWS Secrets Manager wired in via tests/benchmarks/cloudopsbench/infra/,
# and writes to the per-run S3 bucket. The workflow's job is just to
# launch the task and print where to watch.
#
# Trigger from the GitHub UI:
#   Actions → "Benchmark run (manual)" → Run workflow → fill inputs
#
# Pre-reqs (one-time, see tests/benchmarks/cloudopsbench/infra/README.md):
#   - tests/benchmarks/cloudopsbench/infra/ Terraform applied
#   - Bench image pushed to ECR via benchmark-image.yml
#   - Repo secrets seeded into AWS Secrets Manager via benchmark-seed-secret.yml
#   - Repo variables set:
#       AWS_ACCOUNT_ID, BENCH_ECS_CLUSTER, BENCH_TASK_DEFINITION_FAMILY,
#       BENCH_SUBNET_IDS (comma-separated), BENCH_SECURITY_GROUP_ID

on:
  workflow_dispatch:
    inputs:
      config:
        # No adapter-specific default — the operator must point at a
        # specific config. Leaving the field empty surfaces the choice
        # rather than silently dispatching the CloudOpsBench smoke. The
        # path is relative to the container's repo root. Examples:
        #   tests/benchmarks/cloudopsbench/configs/cloudopsbench_smoke.yml
        #   tests/benchmarks/<adapter>/configs/<config>.yml
        description: 'Path to YAML config inside the container (e.g. tests/benchmarks/<adapter>/configs/<config>.yml)'
        required: true
      dev_mode:
        description: 'Dev mode (skip integrity gates, no pre-reg needed)'
        type: boolean
        default: true

# Note: there is intentionally no `image_tag` input here. ECS RunTask
# container overrides do NOT support overriding the image URI — that lives
# on the task definition itself. The image actually pulled is whatever
# was registered the last time `terraform apply -var=image_tag=<tag>` ran
# in tests/benchmarks/cloudopsbench/infra/. Adding a workflow input here would silently mislead
# operators into thinking they control the image per run. To change the
# image, push it via `Benchmark image — build + push to ECR`, then re-apply
# Terraform with the new tag.

permissions:
  contents: read
  id-token: write           # required for AWS OIDC role assumption

concurrency:
  group: benchmark-run
  cancel-in-progress: false

jobs:
  launch:
    name: launch ECS task
    if: github.repository == 'Tracer-Cloud/opensre'
    runs-on: ubuntu-latest
    timeout-minutes: 15     # we only LAUNCH the task; we don't wait for it

    env:
      AWS_REGION: us-east-1

    steps:
      - name: Verify required repo variables
        # Fail loudly BEFORE the AWS auth step if any required repo variable
        # is missing. Without this, an unset var would surface downstream as
        # an opaque error like "Task Definition can not be blank" from
        # describe-task-definition. See tests/benchmarks/cloudopsbench/infra/AWS_BENCH_SETUP.md step 4 for how
        # to set these.
        env:
          AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
          BENCH_ECS_CLUSTER: ${{ vars.BENCH_ECS_CLUSTER }}
          BENCH_TASK_DEFINITION_FAMILY: ${{ vars.BENCH_TASK_DEFINITION_FAMILY }}
          BENCH_SUBNET_IDS: ${{ vars.BENCH_SUBNET_IDS }}
          BENCH_SECURITY_GROUP_ID: ${{ vars.BENCH_SECURITY_GROUP_ID }}
        run: |
          missing=()
          for var in AWS_ACCOUNT_ID BENCH_ECS_CLUSTER BENCH_TASK_DEFINITION_FAMILY \
                     BENCH_SUBNET_IDS BENCH_SECURITY_GROUP_ID; do
            if [ -z "${!var:-}" ]; then
              missing+=("$var")
            fi
          done
          if [ "${#missing[@]}" -gt 0 ]; then
            echo "::error::Missing repo variable(s): ${missing[*]}"
            echo "::error::Set them under Settings → Secrets and variables → Actions → Variables."
            echo "::error::Values come from \`cd tests/benchmarks/cloudopsbench/infra && terraform output\` — see tests/benchmarks/cloudopsbench/infra/AWS_BENCH_SETUP.md step 4."
            exit 1
          fi
          echo "All 5 required repo variables are set."

      - name: Configure AWS credentials (OIDC role assumption)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/opensre-bench-github-actions
          role-session-name: bench-run-${{ github.run_id }}
          aws-region: us-east-1

      - name: Launch ECS RunTask
        # Inputs bound to env vars so the shell does the interpolation —
        # GitHub Actions template syntax (${{ ... }}) is expanded before
        # the shell sees it, which would let a crafted input inject shell
        # metacharacters. Env vars are quoted at use-site.
        env:
          BENCH_CONFIG: ${{ inputs.config }}
          BENCH_DEV_FLAG: ${{ inputs.dev_mode == true && '--dev' || '' }}
          CLUSTER: ${{ vars.BENCH_ECS_CLUSTER }}
          TASK_FAMILY: ${{ vars.BENCH_TASK_DEFINITION_FAMILY }}
          SUBNETS: ${{ vars.BENCH_SUBNET_IDS }}
          SECURITY_GROUP: ${{ vars.BENCH_SECURITY_GROUP_ID }}
        run: |
          set -euo pipefail

          # Surface the image the task definition will actually pull, so
          # operators see the truth (not a workflow input that ECS ignores).
          IMAGE_URI=$(aws ecs describe-task-definition \
            --task-definition "$TASK_FAMILY" \
            --query 'taskDefinition.containerDefinitions[0].image' \
            --output text)

          # Overrides JSON: container env vars passed at runtime. The
          # container's entrypoint reads BENCH_CONFIG + BENCH_DEV_FLAG
          # and invokes:
          #   uv run python -m tests.benchmarks._framework.cli run \
          #     "$BENCH_CONFIG" $BENCH_DEV_FLAG
          OVERRIDES=$(jq -nc \
            --arg config "$BENCH_CONFIG" \
            --arg dev_flag "$BENCH_DEV_FLAG" \
            '{
              containerOverrides: [{
                name: "bench",
                environment: [
                  {name: "BENCH_CONFIG", value: $config},
                  {name: "BENCH_DEV_FLAG", value: $dev_flag}
                ]
              }]
            }')

          TASK_ARN=$(aws ecs run-task \
            --cluster "$CLUSTER" \
            --task-definition "$TASK_FAMILY" \
            --launch-type FARGATE \
            --network-configuration "awsvpcConfiguration={subnets=[$SUBNETS],securityGroups=[$SECURITY_GROUP],assignPublicIp=ENABLED}" \
            --overrides "$OVERRIDES" \
            --query 'tasks[0].taskArn' \
            --output text)

          if [ -z "$TASK_ARN" ] || [ "$TASK_ARN" = "None" ]; then
            echo "::error::run-task returned no taskArn"
            exit 1
          fi

          TASK_ID="${TASK_ARN##*/}"
          echo "task_arn=$TASK_ARN" >> "$GITHUB_OUTPUT"
          echo "task_id=$TASK_ID" >> "$GITHUB_OUTPUT"

          # Job summary — visible in the workflow run page
          {
            echo "## Bench run launched"
            echo ""
            echo "- Image (from task definition): \`$IMAGE_URI\`"
            echo "- Config: \`$BENCH_CONFIG\`"
            echo "- Dev mode: \`${BENCH_DEV_FLAG:-(off)}\`"
            echo "- Task ARN: \`$TASK_ARN\`"
            echo ""
            echo "### Watch it"
            echo ""
            echo "Stream logs locally:"
            echo ""
            echo '```bash'
            echo "aws logs tail /ecs/opensre-bench --follow"
            echo '```'
            echo ""
            echo "Or via AWS Console:"
            echo "https://us-east-1.console.aws.amazon.com/ecs/v2/clusters/$CLUSTER/tasks/$TASK_ID"
            echo ""
            echo "Artifacts land in the bench S3 bucket under \`runs/\` when the task completes."
            echo ""
            echo "_To change the running image: push a new tag via \`Benchmark image — build + push to ECR\`, then \`cd tests/benchmarks/cloudopsbench/infra && terraform apply -var=image_tag=<tag>\`._"
          } >> "$GITHUB_STEP_SUMMARY"