wehub-resource-sync
e768098d0e
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Publish Promptflow Doc / Build (push) Has been cancelled
Publish Promptflow Doc / Deploy (push) Has been cancelled
Flake8 Lint / flake8 (push) Has been cancelled
Spell check CI / Spell_Check (push) Has been cancelled
108 lines
4.2 KiB
YAML
108 lines
4.2 KiB
YAML
name: promptflow-evals-e2e-test-local
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "40 10 * * *" # 2:40 PST every day
|
|
pull_request:
|
|
paths:
|
|
- src/promptflow-evals/**
|
|
- .github/workflows/promptflow-evals-e2e-test-local.yml
|
|
workflow_dispatch:
|
|
|
|
env:
|
|
IS_IN_CI_PIPELINE: "true"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
|
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
|
|
jobs:
|
|
test:
|
|
strategy:
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
python-version: ['3.9', '3.10', '3.11']
|
|
fail-fast: false
|
|
# snok/install-poetry need this to support Windows
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: set test mode
|
|
# Always run in replay mode for now until we figure out the test resource to run live mode
|
|
run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
|
|
#run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install recording
|
|
run: poetry run pip install -e ../promptflow-recording
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install promptflow packages in editable mode
|
|
run: |
|
|
poetry run pip install -e ../promptflow
|
|
poetry run pip install -e ../promptflow-core
|
|
poetry run pip install -e ../promptflow-devkit
|
|
poetry run pip install -e ../promptflow-tracing
|
|
poetry run pip install -e ../promptflow-tools
|
|
poetry run pip install -e ../promptflow-evals
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: generate end-to-end test config from secret
|
|
run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- uses: azure/login@v1
|
|
with:
|
|
client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
|
|
subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
|
|
- name: check azure is not installed
|
|
run: poetry run pytest ../../scripts/code_qa/assert_local_install.py
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run e2e tests
|
|
id: run_e2e_tests_local
|
|
run: |
|
|
poetry run pytest -m localtest tests/evals/e2etests --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
|
|
poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity e2e_tests_local --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
|
|
|
report:
|
|
needs: test
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
checks: write
|
|
pull-requests: write
|
|
contents: read
|
|
issues: read
|
|
steps:
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
path: artifacts
|
|
- uses: EnricoMi/publish-unit-test-result-action@v2
|
|
with:
|
|
check_name: promptflow-evals test result
|
|
comment_title: promptflow-evals test result
|
|
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
|
- uses: irongut/CodeCoverageSummary@v1.3.0
|
|
with:
|
|
filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
|
|
badge: true
|
|
fail_below_min: false
|
|
format: markdown
|
|
hide_complexity: true
|
|
output: both
|
|
thresholds: 40 80
|