promptfoo--promptfoo/.github/workflows/docker.yml

# This GitHub Action builds the Docker image Promptfoo,
# runs the Docker container, performs a health check to ensure the application
# is running correctly, and then publishes the image to GitHub Container Registry
# if all checks pass. This action is triggered by the release-please workflow,
# published human-created releases, workflow dispatch, pull requests to main that
# modify the Docker/runtime update contract, and matching pushes to main.

name: Test and Publish Multi-arch Docker Image
run-name: |
  ${{ (inputs.tag_name || github.event.release.tag_name) && 'Release' ||
      github.event_name == 'workflow_dispatch' && 'Manual' ||
      github.event_name == 'pull_request' && 'PR' ||
      'Push' }}
  Docker build for ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }}
  ${{ github.event_name == 'pull_request' && format('(PR #{0})', github.event.number) || '' }}
  by @${{ github.actor }}

on:
  workflow_call:
    inputs:
      tag_name:
        description: 'The release tag name (e.g., 0.120.0)'
        required: true
        type: string
  workflow_dispatch:
    inputs:
      tag_name:
        description: 'Optional release tag to publish (e.g., 0.121.6)'
        required: false
        type: string
  release:
    types:
      - published
  pull_request:
    branches:
      - main
    paths:
      - '.github/workflows/docker.yml'
      - 'Dockerfile'
      - 'src/envars.ts'
      - 'src/globalConfig/runtimeNoticeState.ts'
      - 'src/runtimeCompatibility*.ts'
      - 'src/server/routes/version*.ts'
      - 'src/types/api/version.ts'
      - 'src/types/runtimeCompatibility.ts'
      - 'src/updates.ts'
      - 'src/updates/**'
  push:
    branches:
      - main
    paths:
      - '.github/workflows/docker.yml'
      - 'Dockerfile'
      - 'src/envars.ts'
      - 'src/globalConfig/runtimeNoticeState.ts'
      - 'src/runtimeCompatibility*.ts'
      - 'src/server/routes/version*.ts'
      - 'src/types/api/version.ts'
      - 'src/types/runtimeCompatibility.ts'
      - 'src/updates.ts'
      - 'src/updates/**'

concurrency:
  # Reusable workflow calls inherit the caller's github.workflow value. Keep this
  # group Docker-specific so release-please callers are not cancelled mid-release.
  group: ${{ github.event_name == 'pull_request' && format('docker-pr-{0}', github.ref) || format('docker-publish-{0}', github.repository) }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  RELEASE_TAG: ${{ inputs.tag_name || github.event.release.tag_name || '' }}
  SOURCE_REF: ${{ inputs.tag_name || github.event.release.tag_name || github.ref }}
  BUILD_VERSION: ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }}
  BUILD_DATE: ${{ github.event_name == 'pull_request' && github.event.pull_request.updated_at || github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp || github.event.repository.updated_at }}
  # Fallback release paths build a specific tag outside release-please.yml.
  IS_RELEASE_FALLBACK: ${{ (github.event_name == 'workflow_dispatch' && inputs.tag_name != '') || github.event_name == 'release' }}

permissions:
  contents: read

jobs:
  test:
    # Bot-authored releases are already handled by release-please.yml after npm publish.
    # The release event path is for human-published fallback releases and backfills.
    if: github.event_name != 'release' || github.event.release.author.type != 'Bot'
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: read

    steps:
      - name: Validate release tag
        if: github.event_name == 'release' || env.RELEASE_TAG != ''
        run: |
          if [[ ! "$RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]]; then
            echo "::error::RELEASE_TAG '$RELEASE_TAG' is empty or invalid"
            exit 1
          fi

      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo apt-get clean
          df -h

      - name: Checkout repository
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          ref: ${{ env.SOURCE_REF }}
          persist-credentials: false

      # github.sha is the trigger SHA; SOURCE_REF can point checkout at a different tag.
      - name: Resolve source commit
        id: source
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4

      - name: Build official Docker image for testing
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
        with:
          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-linux-amd64
          context: .
          push: false
          load: true
          tags: local-test-image:official
          platforms: linux/amd64
          build-args: |
            BUILD_VERSION=${{ env.BUILD_VERSION }}
            BUILD_DATE=${{ env.BUILD_DATE }}
            GIT_SHA=${{ steps.source.outputs.sha }}
            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=1

      - name: Build custom Docker image for contract testing
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
        with:
          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-linux-amd64
          context: .
          push: false
          load: true
          tags: local-test-image:custom
          platforms: linux/amd64
          build-args: |
            BUILD_VERSION=${{ env.BUILD_VERSION }}
            BUILD_DATE=${{ env.BUILD_DATE }}
            GIT_SHA=${{ steps.source.outputs.sha }}
            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=0

      - name: Run Docker containers for testing
        run: |
          docker run -d --name promptfoo-container -p 3000:3000 local-test-image:official
          docker run -d --name promptfoo-custom-container -p 3001:3000 local-test-image:custom

      - name: Run health check
        run: |
          # Wait for both servers to be fully ready (including database migrations)
          # before running any CLI commands that also trigger migrations.
          for port in 3000 3001; do
            HEALTH_CHECK_PASS=
            for ((retry=1; retry<=10; retry++)); do
              if curl -f "http://localhost:${port}/health"; then
                echo -e "\nHealth check passed on port ${port}"
                HEALTH_CHECK_PASS=true
                break
              else
                echo "Health check on port ${port} failed, retrying in 2 seconds..."
                sleep 2
              fi
            done

            if [ -z "$HEALTH_CHECK_PASS" ]; then
              echo -e "\nHealth check on port ${port} failed after multiple attempts" >&2
              exit 1
            fi
          done

      - name: Verify Python and promptfoo
        run: |
          echo "Checking Python version:"
          if ! docker exec promptfoo-container python --version; then
            echo "Python check failed"
            exit 1
          fi

          echo "Checking promptfoo version:"
          if ! docker exec promptfoo-container promptfoo --version; then
            echo "promptfoo check failed"
            exit 1
          fi

          echo "Checking pf alias:"
          if ! docker exec promptfoo-container pf --version; then
            echo "pf alias check failed"
            exit 1
          fi

          echo "Checking container update markers and API guidance:"
          test "$(docker exec promptfoo-container printenv PROMPTFOO_RUNNING_IN_DOCKER)" = "1"
          test "$(docker exec promptfoo-container printenv PROMPTFOO_OFFICIAL_DOCKER_IMAGE)" = "1"
          test "$(docker exec promptfoo-custom-container printenv PROMPTFOO_RUNNING_IN_DOCKER)" = "1"
          test "$(docker exec promptfoo-custom-container printenv PROMPTFOO_OFFICIAL_DOCKER_IMAGE)" = "0"

          curl -fsS http://localhost:3000/api/version | jq -e '
            .commandType == "docker" and
            .updateCommands.primary == "docker pull ghcr.io/promptfoo/promptfoo:latest"
          '
          curl -fsS http://localhost:3001/api/version | jq -e '
            .commandType == "npm" and
            .updateCommands.primary == "" and
            .updateCommands.isCustomContainer == true
          '

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669)
      - name: Upgrade npm
        run: npm install -g npm@11.11.0 --registry=https://registry.npmjs.org/

      - name: run promptfoo eval
        id: eval
        env:
          PROMPTFOO_REMOTE_API_BASE_URL: http://localhost:3000
          PROMPTFOO_SHARING_APP_BASE_URL: http://localhost:3000
        run: |
          npm install --registry=https://registry.npmjs.org/
          npm run local -- eval -c .github/assets/promptfooconfig.yaml --share

      - name: Test that the eval results are uploaded
        run: |
          response=$(curl -s http://localhost:3000/api/results)
          echo "Response: $response"

          # Use jq to extract the array length
          count=$(echo "$response" | jq '.data | length')
          echo "Array Length: $count"

          # Check if the count is exactly 1
          if [ "$count" -ne 1 ]; then
            echo "Error: Expected 1 entry, but got $count"
            exit 1
          fi

      - name: Stop and remove Docker container
        if: always()
        run: docker rm -f promptfoo-container promptfoo-custom-container || true

      - name: Clean up Docker images
        if: always()
        run: |
          docker system prune -af
          docker volume prune -f
          df -h

  build-docker-and-push-digests:
    if: (github.event_name != 'pull_request')
    strategy:
      matrix:
        platforms:
          [
            { runner: ubuntu-latest, platform: linux/amd64, digest-suffix: linux-amd64 },
            { runner: ubuntu-24.04-arm, platform: linux/arm64, digest-suffix: linux-arm64 },
          ]
    runs-on: ${{ matrix.platforms.runner }}
    timeout-minutes: 60
    needs: [test]
    permissions:
      contents: read
      packages: write
    steps:
      - name: Validate release tag
        if: github.event_name == 'release' || env.RELEASE_TAG != ''
        run: |
          if [[ ! "$RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]]; then
            echo "::error::RELEASE_TAG '$RELEASE_TAG' is empty or invalid"
            exit 1
          fi

      - name: Checkout repository
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          ref: ${{ env.SOURCE_REF }}

      # github.sha is the trigger SHA; SOURCE_REF can point checkout at a different tag.
      - name: Resolve source commit
        id: source
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          labels: |
            org.opencontainers.image.revision=${{ steps.source.outputs.sha }}

      - name: Log in to the Container registry
        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push multi-arch Docker image
        id: build
        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
        with:
          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platforms.digest-suffix }}
          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platforms.digest-suffix }},mode=max
          context: .
          platforms: ${{ matrix.platforms.platform }}
          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            BUILD_VERSION=${{ env.BUILD_VERSION }}
            BUILD_DATE=${{ env.BUILD_DATE }}
            GIT_SHA=${{ steps.source.outputs.sha }}
            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=${{ github.repository == 'promptfoo/promptfoo' && '1' || '0' }}
          outputs: |
            type=image,push-by-digest=true,name-canonical=true,push=true,annotation-index.org.opencontainers.image.description=promptfoo is a tool for testing evaluating and red-teaming LLM apps.

      - name: Export digest
        run: |
          mkdir -p "${{ runner.temp }}/digests"
          digest="${{ steps.build.outputs.digest }}"
          touch "${{ runner.temp }}/digests/${digest#sha256:}"

      - name: Upload digest
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: digests-${{ matrix.platforms.digest-suffix }}
          path: ${{ runner.temp }}/digests/*
          if-no-files-found: error
          retention-days: 1

  merge-docker-digests:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs: [build-docker-and-push-digests]
    permissions:
      packages: write
    outputs:
      manifest_digest: ${{ steps.inspect.outputs.digest }}
    steps:
      - name: Download digests
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          path: ${{ runner.temp }}/digests
          pattern: digests-*
          merge-multiple: true

      - name: Log in to the Container registry
        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          # Fallback release builds publish immutable version tags only so moving aliases stay on normal release paths.
          tags: |
            type=ref,event=branch,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
            type=ref,event=pr,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}},enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
            type=sha,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
            type=raw,value=${{ env.RELEASE_TAG }},enable=${{ env.RELEASE_TAG != '' }}
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && env.IS_RELEASE_FALLBACK != 'true' }}

      - name: Create manifest list and push
        working-directory: ${{ runner.temp }}/digests
        run: |
          # shellcheck disable=SC2046,SC2091
          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)

      - name: Inspect image
        id: inspect
        env:
          IMAGE_TAG: ${{ env.RELEASE_TAG || steps.meta.outputs.version }}
        run: |
          IMAGE_REF="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}"
          # Log full inspection details and extract digest
          INSPECT_OUTPUT=$(docker buildx imagetools inspect "${IMAGE_REF}")
          echo "$INSPECT_OUTPUT"
          # Extract the manifest digest (first Digest: line)
          DIGEST=$(echo "$INSPECT_OUTPUT" | grep -m1 "^Digest:" | awk '{print $2}')
          if [ -z "$DIGEST" ]; then
            echo "Error: Failed to extract manifest digest from inspect output"
            exit 1
          fi
          echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
          echo "Manifest digest: $DIGEST"

  attest-docker-image:
    name: 'Attest Multi-arch Image'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs: [merge-docker-digests]
    permissions:
      id-token: write
      attestations: write
      packages: write

    steps:
      - name: Log in to the Container registry
        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Attest Build Provenance
        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4
        with:
          subject-name: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}'
          subject-digest: '${{ needs.merge-docker-digests.outputs.manifest_digest }}'
          push-to-registry: true