microsoft--promptflow/.github/workflows/promptflow-core-test.yml

name: promptflow-core-test [Pure]

on:
  schedule:
    - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
  pull_request:
    paths:
      - src/promptflow-tracing/**
      - src/promptflow-core/**
      - .github/workflows/promptflow-core-test.yml
  workflow_dispatch:

env:
  IS_IN_CI_PIPELINE: "true"
  PF_DISABLE_TRACING: "false"
  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow

permissions:
  id-token: write
  contents: read

jobs:
  core_test:
    environment:
      internal
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.9', '3.10', '3.11']
      fail-fast: false
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - uses: snok/install-poetry@v1
      - name: install test dependency group
        run: |
          poetry install -E executor-service --with ci,test
          poetry run pip show promptflow-tracing
          poetry run pip show promptflow-core
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: generate end-to-end test config from secret
        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: set test mode
        run: |
          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
      - name: Azure login (non pull_request workflow)
        if: github.event_name != 'pull_request'
        uses: azure/login@v1
        with:
          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
          tenant-id: ${{secrets.AZURE_TENANT_ID}}
          client-id: ${{secrets.AZURE_CLIENT_ID}}
      - name: generate live test resources (non pull_request workflow)
        if: github.event_name != 'pull_request'
        uses: "./.github/actions/step_generate_configs"
        with:
          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
      - name: generate live test resources (pull_request workflow)
        if: github.event_name == 'pull_request'
        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
        run: |
          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
      - name: run core tests
        run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/core/*.xml
            ${{ env.WORKING_DIRECTORY }}/core/htmlcov/

  azureml_serving_test:
    environment:
      internal
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.9', '3.10', '3.11']
      fail-fast: false
    # snok/install-poetry need this to support Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - uses: snok/install-poetry@v1
      - name: install test dependency group
        run: |
          poetry install -E azureml-serving --with ci,test
          poetry run pip show promptflow-tracing
          poetry run pip show promptflow-core
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: set test mode
        run: |
          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
      - name: Azure login (non pull_request workflow)
        if: github.event_name != 'pull_request'
        uses: azure/login@v1
        with:
          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
          tenant-id: ${{secrets.AZURE_TENANT_ID}}
          client-id: ${{secrets.AZURE_CLIENT_ID}}
      - name: generate live test resources (non pull_request workflow)
        if: github.event_name != 'pull_request'
        uses: "./.github/actions/step_generate_configs"
        with:
          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
      - name: generate live test resources (pull_request workflow)
        if: github.event_name == 'pull_request'
        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
        run: |
          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
      - name: run azureml-serving tests
        run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
        working-directory: ${{ env.WORKING_DIRECTORY }}
      - name: upload coverage report
        uses: actions/upload-artifact@v4
        with:
          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
          path: |
            ${{ env.WORKING_DIRECTORY }}/azureml-serving/*.xml
            ${{ env.WORKING_DIRECTORY }}/azureml-serving/htmlcov/

  report:
    needs: [core_test, azureml_serving_test]
    runs-on: ubuntu-latest
    permissions:
      checks: write
      pull-requests: write
      contents: read
      issues: read
    steps:
      - uses: actions/download-artifact@v4
        with:
          path: artifacts
      - uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          check_name: promptflow-core test result
          comment_title: promptflow-core test result
          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
#     TODO: Enable coverage check after core test fully setup
#      - uses: irongut/CodeCoverageSummary@v1.3.0
#        with:
#          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
#          badge: true
#          fail_below_min: true
#          format: markdown
#          hide_complexity: true
#          output: both
#          thresholds: 40 60