comet-ml--opik/.github/workflows/python_sdk_e2e_tests.yml

name: Python SDK E2E Tests
run-name: "Python SDK E2E Tests ${{ github.ref_name }} by @${{ github.actor }}"
permissions:
  contents: read
  checks: write
  pull-requests: write
  packages: write
on:
    workflow_dispatch:
    pull_request:
        paths:
          - 'sdks/python/**'
          - 'apps/opik-backend/**'
          - '.github/workflows/python_sdk_e2e_tests.yml'
    push:
        branches:
          - 'main'
        paths:
          - 'sdks/python/**'
          - 'apps/opik-backend/**'
          - '.github/workflows/python_sdk_e2e_tests.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
  OPIK_ENABLE_LITELLM_MODELS_MONITORING: False
  OPIK_SENTRY_ENABLE: False
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }}
  OPIK_URL_OVERRIDE: http://localhost:8080
  OPIK_CONSOLE_LOGGING_LEVEL: INFO
jobs:
    select-matrix:
        name: Select Python version matrix
        runs-on: ubuntu-latest
        timeout-minutes: 2
        outputs:
            python_versions: ${{ steps.pick.outputs.versions }}
        steps:
            - uses: actions/checkout@v6
              with:
                  sparse-checkout: .github/actions/select-e2e-matrix
                  sparse-checkout-cone-mode: false
            - uses: ./.github/actions/select-e2e-matrix
              id: pick
              with:
                  all-versions: ${{ vars.PYTHON_VERSIONS }}
                  match-regex: '^sdks/python/|^\.github/workflows/python_sdk_e2e_tests\.yml$'
                  github-token: ${{ secrets.GITHUB_TOKEN }}

    build-opik:
        uses: ./.github/workflows/build_e2e_docker.yaml
        with:
          include_guardrails: true

    run-e2e:
        needs: [select-matrix, build-opik]
        name: Python SDK E2E Tests ${{matrix.python_version}}
        runs-on: ubuntu-latest
        timeout-minutes: 30
        strategy:
            fail-fast: false
            matrix:
                python_version: ${{ fromJSON(needs.select-matrix.outputs.python_versions) }}

        steps:
        - name: Checkout
          uses: actions/checkout@v6

        - name: Login to GHCR
          uses: docker/login-action@v3
          with:
            registry: ghcr.io
            username: ${{ github.actor }}
            password: ${{ secrets.GITHUB_TOKEN }}

        - name: Pull Docker images from GHCR
          env:
            TAG: ${{ needs.build-opik.outputs.image_tag }}
          run: |
            docker pull "ghcr.io/comet-ml/opik/opik-backend:$TAG"
            docker pull "ghcr.io/comet-ml/opik/opik-python-backend:$TAG"
            docker pull "ghcr.io/comet-ml/opik/opik-guardrails-backend:$TAG"
            docker tag "ghcr.io/comet-ml/opik/opik-backend:$TAG" ghcr.io/comet-ml/opik/opik-backend:latest
            docker tag "ghcr.io/comet-ml/opik/opik-python-backend:$TAG" ghcr.io/comet-ml/opik/opik-python-backend:latest
            docker tag "ghcr.io/comet-ml/opik/opik-guardrails-backend:$TAG" ghcr.io/comet-ml/opik/opik-guardrails-backend:latest

        - name: Setup Python ${{matrix.python_version}}
          uses: actions/setup-python@v5
          with:
            python-version: ${{matrix.python_version}}

        - name: Run latest Opik server
          env:
            OPIK_USAGE_REPORT_ENABLED: false
            # Avoid Buildx Bake concurrent image export collisions in CI.
            COMPOSE_BAKE: false
            TOGGLE_RUNNERS_ENABLED: "true"
            OPIK_BACKEND_PULL_POLICY: never
            PYTHON_BACKEND_PULL_POLICY: never
            OPIK_GUARDRAILS_BACKEND_PULL_POLICY: never
          run: |
            cd ${{ github.workspace }}
            ./opik.sh --backend --port-mapping --guardrails

        - name: Check Opik server availability
          shell: bash
          run: |
            chmod +x ${{ github.workspace }}/tests_end_to_end/installer_utils/*.sh
            cd ${{ github.workspace }}/deployment/docker-compose
            echo "Check Docker pods are up"
            ${{ github.workspace }}/tests_end_to_end/installer_utils/check_docker_compose_pods.sh
            echo "Check backend health"
            ${{ github.workspace }}/tests_end_to_end/installer_utils/check_backend.sh

        - name: Install opik SDK
          run: |
            cd ${{ github.workspace }}/sdks/python
            pip install .

        - name: Run smoke tests
          run: |
            cd ${{ github.workspace }}/sdks/python/tests/e2e_smoke
            ./smoke_tests_runner.sh

        - name: Install test requirements
          run: |
            cd ${{ github.workspace }}/sdks/python
            pip install -r tests/test_requirements.txt
            pip list

        - name: Run tests
          # -n 3 + --dist=loadfile distributes whole test files across three
          # worker processes (one file per worker). Past ~3 the gains
          # flatten — we become tail-bound on the slowest single file and
          # the docker-compose backend on the same runner starts to thrash.
          # E2E isolation hinges on --dist=loadfile: the per-module project
          # name (generate_project_name("e2e", __name__)) is only collision-
          # free if a single file is not split across workers.
          # -p no:benchmark silences pytest-benchmark's auto-disable warning
          # under xdist.
          run: |
            cd ${{ github.workspace }}/sdks/python
            pytest tests/e2e --ignore=tests/e2e/test_guardrails.py --ignore=tests/e2e/compatibility_v1 -vv -n 3 --dist=loadfile -p no:benchmark --durations=20 --junitxml=${{ github.workspace }}/test_results_${{matrix.python_version}}.xml

        - name: Publish Test Report
          uses: EnricoMi/publish-unit-test-result-action/linux@v2
          if: always()
          with:
            action_fail: true
            comment_mode: failures
            check_name: Python SDK E2E Tests Results (Python ${{matrix.python_version}})
            files: ${{ github.workspace }}/test_results_${{matrix.python_version}}.xml

        - name: Keep BE log in case of failure
          if: failure()
          run: |
            docker logs opik-backend-1 > ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log

        - name: Attach BE log
          if: failure()
          uses: actions/upload-artifact@v7
          with:
              name: opik-backend-log-p${{matrix.python_version}}
              path: ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log

        - name: Stop opik server
          if: always()
          run: |
            cd ${{ github.workspace }}
            ./opik.sh --stop --guardrails