chore: import upstream snapshot with attribution

2026-07-13 13:24:08 +08:00
commit 0d3cb498a3
5438 changed files with 1316560 additions and 0 deletions
@@ -0,0 +1,443 @@
+# This GitHub Action builds the Docker image Promptfoo,
+# runs the Docker container, performs a health check to ensure the application
+# is running correctly, and then publishes the image to GitHub Container Registry
+# if all checks pass. This action is triggered by the release-please workflow,
+# published human-created releases, workflow dispatch, pull requests to main that
+# modify the Docker/runtime update contract, and matching pushes to main.
+
+name: Test and Publish Multi-arch Docker Image
+run-name: |
+  ${{ (inputs.tag_name || github.event.release.tag_name) && 'Release' ||
+      github.event_name == 'workflow_dispatch' && 'Manual' ||
+      github.event_name == 'pull_request' && 'PR' ||
+      'Push' }}
+  Docker build for ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }}
+  ${{ github.event_name == 'pull_request' && format('(PR #{0})', github.event.number) || '' }}
+  by @${{ github.actor }}
+
+on:
+  workflow_call:
+    inputs:
+      tag_name:
+        description: 'The release tag name (e.g., 0.120.0)'
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Optional release tag to publish (e.g., 0.121.6)'
+        required: false
+        type: string
+  release:
+    types:
+      - published
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/docker.yml'
+      - 'Dockerfile'
+      - 'src/envars.ts'
+      - 'src/globalConfig/runtimeNoticeState.ts'
+      - 'src/runtimeCompatibility*.ts'
+      - 'src/server/routes/version*.ts'
+      - 'src/types/api/version.ts'
+      - 'src/types/runtimeCompatibility.ts'
+      - 'src/updates.ts'
+      - 'src/updates/**'
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/docker.yml'
+      - 'Dockerfile'
+      - 'src/envars.ts'
+      - 'src/globalConfig/runtimeNoticeState.ts'
+      - 'src/runtimeCompatibility*.ts'
+      - 'src/server/routes/version*.ts'
+      - 'src/types/api/version.ts'
+      - 'src/types/runtimeCompatibility.ts'
+      - 'src/updates.ts'
+      - 'src/updates/**'
+
+concurrency:
+  # Reusable workflow calls inherit the caller's github.workflow value. Keep this
+  # group Docker-specific so release-please callers are not cancelled mid-release.
+  group: ${{ github.event_name == 'pull_request' && format('docker-pr-{0}', github.ref) || format('docker-publish-{0}', github.repository) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  RELEASE_TAG: ${{ inputs.tag_name || github.event.release.tag_name || '' }}
+  SOURCE_REF: ${{ inputs.tag_name || github.event.release.tag_name || github.ref }}
+  BUILD_VERSION: ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }}
+  BUILD_DATE: ${{ github.event_name == 'pull_request' && github.event.pull_request.updated_at || github.event_name == 'release' && github.event.release.published_at || github.event.head_commit.timestamp || github.event.repository.updated_at }}
+  # Fallback release paths build a specific tag outside release-please.yml.
+  IS_RELEASE_FALLBACK: ${{ (github.event_name == 'workflow_dispatch' && inputs.tag_name != '') || github.event_name == 'release' }}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    # Bot-authored releases are already handled by release-please.yml after npm publish.
+    # The release event path is for human-published fallback releases and backfills.
+    if: github.event_name != 'release' || github.event.release.author.type != 'Bot'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+
+    steps:
+      - name: Validate release tag
+        if: github.event_name == 'release' || env.RELEASE_TAG != ''
+        run: |
+          if [[ ! "$RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]]; then
+            echo "::error::RELEASE_TAG '$RELEASE_TAG' is empty or invalid"
+            exit 1
+          fi
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo apt-get clean
+          df -h
+
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ env.SOURCE_REF }}
+          persist-credentials: false
+
+      # github.sha is the trigger SHA; SOURCE_REF can point checkout at a different tag.
+      - name: Resolve source commit
+        id: source
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
+
+      - name: Build official Docker image for testing
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
+        with:
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-linux-amd64
+          context: .
+          push: false
+          load: true
+          tags: local-test-image:official
+          platforms: linux/amd64
+          build-args: |
+            BUILD_VERSION=${{ env.BUILD_VERSION }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            GIT_SHA=${{ steps.source.outputs.sha }}
+            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=1
+
+      - name: Build custom Docker image for contract testing
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
+        with:
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-linux-amd64
+          context: .
+          push: false
+          load: true
+          tags: local-test-image:custom
+          platforms: linux/amd64
+          build-args: |
+            BUILD_VERSION=${{ env.BUILD_VERSION }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            GIT_SHA=${{ steps.source.outputs.sha }}
+            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=0
+
+      - name: Run Docker containers for testing
+        run: |
+          docker run -d --name promptfoo-container -p 3000:3000 local-test-image:official
+          docker run -d --name promptfoo-custom-container -p 3001:3000 local-test-image:custom
+
+      - name: Run health check
+        run: |
+          # Wait for both servers to be fully ready (including database migrations)
+          # before running any CLI commands that also trigger migrations.
+          for port in 3000 3001; do
+            HEALTH_CHECK_PASS=
+            for ((retry=1; retry<=10; retry++)); do
+              if curl -f "http://localhost:${port}/health"; then
+                echo -e "\nHealth check passed on port ${port}"
+                HEALTH_CHECK_PASS=true
+                break
+              else
+                echo "Health check on port ${port} failed, retrying in 2 seconds..."
+                sleep 2
+              fi
+            done
+
+            if [ -z "$HEALTH_CHECK_PASS" ]; then
+              echo -e "\nHealth check on port ${port} failed after multiple attempts" >&2
+              exit 1
+            fi
+          done
+
+      - name: Verify Python and promptfoo
+        run: |
+          echo "Checking Python version:"
+          if ! docker exec promptfoo-container python --version; then
+            echo "Python check failed"
+            exit 1
+          fi
+
+          echo "Checking promptfoo version:"
+          if ! docker exec promptfoo-container promptfoo --version; then
+            echo "promptfoo check failed"
+            exit 1
+          fi
+
+          echo "Checking pf alias:"
+          if ! docker exec promptfoo-container pf --version; then
+            echo "pf alias check failed"
+            exit 1
+          fi
+
+          echo "Checking container update markers and API guidance:"
+          test "$(docker exec promptfoo-container printenv PROMPTFOO_RUNNING_IN_DOCKER)" = "1"
+          test "$(docker exec promptfoo-container printenv PROMPTFOO_OFFICIAL_DOCKER_IMAGE)" = "1"
+          test "$(docker exec promptfoo-custom-container printenv PROMPTFOO_RUNNING_IN_DOCKER)" = "1"
+          test "$(docker exec promptfoo-custom-container printenv PROMPTFOO_OFFICIAL_DOCKER_IMAGE)" = "0"
+
+          curl -fsS http://localhost:3000/api/version | jq -e '
+            .commandType == "docker" and
+            .updateCommands.primary == "docker pull ghcr.io/promptfoo/promptfoo:latest"
+          '
+          curl -fsS http://localhost:3001/api/version | jq -e '
+            .commandType == "npm" and
+            .updateCommands.primary == "" and
+            .updateCommands.isCustomContainer == true
+          '
+
+      - name: Use Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669)
+      - name: Upgrade npm
+        run: npm install -g npm@11.11.0 --registry=https://registry.npmjs.org/
+
+      - name: run promptfoo eval
+        id: eval
+        env:
+          PROMPTFOO_REMOTE_API_BASE_URL: http://localhost:3000
+          PROMPTFOO_SHARING_APP_BASE_URL: http://localhost:3000
+        run: |
+          npm install --registry=https://registry.npmjs.org/
+          npm run local -- eval -c .github/assets/promptfooconfig.yaml --share
+
+      - name: Test that the eval results are uploaded
+        run: |
+          response=$(curl -s http://localhost:3000/api/results)
+          echo "Response: $response"
+
+          # Use jq to extract the array length
+          count=$(echo "$response" | jq '.data | length')
+          echo "Array Length: $count"
+
+          # Check if the count is exactly 1
+          if [ "$count" -ne 1 ]; then
+            echo "Error: Expected 1 entry, but got $count"
+            exit 1
+          fi
+
+      - name: Stop and remove Docker container
+        if: always()
+        run: docker rm -f promptfoo-container promptfoo-custom-container || true
+
+      - name: Clean up Docker images
+        if: always()
+        run: |
+          docker system prune -af
+          docker volume prune -f
+          df -h
+
+  build-docker-and-push-digests:
+    if: (github.event_name != 'pull_request')
+    strategy:
+      matrix:
+        platforms:
+          [
+            { runner: ubuntu-latest, platform: linux/amd64, digest-suffix: linux-amd64 },
+            { runner: ubuntu-24.04-arm, platform: linux/arm64, digest-suffix: linux-arm64 },
+          ]
+    runs-on: ${{ matrix.platforms.runner }}
+    timeout-minutes: 60
+    needs: [test]
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Validate release tag
+        if: github.event_name == 'release' || env.RELEASE_TAG != ''
+        run: |
+          if [[ ! "$RELEASE_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?$ ]]; then
+            echo "::error::RELEASE_TAG '$RELEASE_TAG' is empty or invalid"
+            exit 1
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ env.SOURCE_REF }}
+
+      # github.sha is the trigger SHA; SOURCE_REF can point checkout at a different tag.
+      - name: Resolve source commit
+        id: source
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: |
+            org.opencontainers.image.revision=${{ steps.source.outputs.sha }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push multi-arch Docker image
+        id: build
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
+        with:
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platforms.digest-suffix }}
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platforms.digest-suffix }},mode=max
+          context: .
+          platforms: ${{ matrix.platforms.platform }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            BUILD_VERSION=${{ env.BUILD_VERSION }}
+            BUILD_DATE=${{ env.BUILD_DATE }}
+            GIT_SHA=${{ steps.source.outputs.sha }}
+            PROMPTFOO_OFFICIAL_DOCKER_IMAGE=${{ github.repository == 'promptfoo/promptfoo' && '1' || '0' }}
+          outputs: |
+            type=image,push-by-digest=true,name-canonical=true,push=true,annotation-index.org.opencontainers.image.description=promptfoo is a tool for testing evaluating and red-teaming LLM apps.
+
+      - name: Export digest
+        run: |
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: digests-${{ matrix.platforms.digest-suffix }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-docker-digests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs: [build-docker-and-push-digests]
+    permissions:
+      packages: write
+    outputs:
+      manifest_digest: ${{ steps.inspect.outputs.digest }}
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          # Fallback release builds publish immutable version tags only so moving aliases stay on normal release paths.
+          tags: |
+            type=ref,event=branch,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
+            type=ref,event=pr,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
+            type=sha,enable=${{ env.IS_RELEASE_FALLBACK != 'true' }}
+            type=raw,value=${{ env.RELEASE_TAG }},enable=${{ env.RELEASE_TAG != '' }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && env.IS_RELEASE_FALLBACK != 'true' }}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          # shellcheck disable=SC2046,SC2091
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        id: inspect
+        env:
+          IMAGE_TAG: ${{ env.RELEASE_TAG || steps.meta.outputs.version }}
+        run: |
+          IMAGE_REF="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}"
+          # Log full inspection details and extract digest
+          INSPECT_OUTPUT=$(docker buildx imagetools inspect "${IMAGE_REF}")
+          echo "$INSPECT_OUTPUT"
+          # Extract the manifest digest (first Digest: line)
+          DIGEST=$(echo "$INSPECT_OUTPUT" | grep -m1 "^Digest:" | awk '{print $2}')
+          if [ -z "$DIGEST" ]; then
+            echo "Error: Failed to extract manifest digest from inspect output"
+            exit 1
+          fi
+          echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
+          echo "Manifest digest: $DIGEST"
+
+  attest-docker-image:
+    name: 'Attest Multi-arch Image'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs: [merge-docker-digests]
+    permissions:
+      id-token: write
+      attestations: write
+      packages: write
+
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Attest Build Provenance
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4
+        with:
+          subject-name: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}'
+          subject-digest: '${{ needs.merge-docker-digests.outputs.manifest_digest }}'
+          push-to-registry: true