wehub-resource-sync
e768098d0e
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Publish Promptflow Doc / Build (push) Has been cancelled
Publish Promptflow Doc / Deploy (push) Has been cancelled
Flake8 Lint / flake8 (push) Has been cancelled
Spell check CI / Spell_Check (push) Has been cancelled
469 lines
19 KiB
YAML
469 lines
19 KiB
YAML
name: promptflow-release-testing-matrix
|
|
on:
|
|
workflow_call:
|
|
workflow_dispatch:
|
|
inputs:
|
|
# can leave empty when trigger manually
|
|
# GitHub Actions API for trigger does not return workflow run id
|
|
# there we reference below Stack Overflow solution:
|
|
# https://stackoverflow.com/a/69500478
|
|
# which adds an identifier in workflow run jobs and can be used for filter
|
|
id:
|
|
description: Identifier for the workflow run
|
|
required: false
|
|
type: string
|
|
env:
|
|
IS_IN_CI_PIPELINE: "true"
|
|
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
|
AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
|
|
CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
|
DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
|
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
|
|
TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
|
|
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
|
PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
|
|
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
|
|
jobs:
|
|
id:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: workflow run id - ${{ inputs.id }}
|
|
run: |
|
|
echo "workflow run id: ${{ inputs.id }}"
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.9"
|
|
- uses: snok/install-poetry@v1
|
|
- working-directory: ${{ env.TRACING_DIRECTORY }}
|
|
run: poetry build -f wheel
|
|
- working-directory: ${{ env.CORE_DIRECTORY }}
|
|
run: poetry build -f wheel
|
|
- working-directory: ${{ env.DEVKIT_DIRECTORY }}
|
|
run: poetry build -f wheel
|
|
- working-directory: ${{ env.AZURE_DIRECTORY }}
|
|
run: poetry build -f wheel
|
|
- working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
|
|
run: |
|
|
pip install -r ./dev_requirements.txt
|
|
python ./setup.py bdist_wheel
|
|
- working-directory: ${{ env.TOOL_DIRECTORY }}
|
|
run: python ./setup.py bdist_wheel
|
|
|
|
- name: Upload Wheel
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: |
|
|
${{ github.workspace }}/src/promptflow/dist/*.whl
|
|
${{ github.workspace }}/src/promptflow-tracing/dist/*.whl
|
|
${{ github.workspace }}/src/promptflow-core/dist/*.whl
|
|
${{ github.workspace }}/src/promptflow-devkit/dist/*.whl
|
|
${{ github.workspace }}/src/promptflow-azure/dist/*.whl
|
|
${{ github.workspace }}/src/promptflow-tools/dist/*.whl
|
|
|
|
promptflow_tracing_tests:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
|
needs: build
|
|
env:
|
|
PROMPT_FLOW_TEST_MODE: "live"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@v4
|
|
- name: Display and Set Environment Variables
|
|
run:
|
|
env | sort >> $GITHUB_OUTPUT
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.pythonVersion }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
|
- name: install promptflow-tracing from wheel
|
|
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
|
run: poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install recording
|
|
run: poetry install
|
|
working-directory: ${{ env.RECORD_DIRECTORY }}
|
|
- name: generate end-to-end test config from secret
|
|
run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run e2e tests
|
|
run: poetry run pytest -m e2etest --tb=short
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: promptflow_tracing_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
|
|
|
|
promptflow_core_tests:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
|
needs: build
|
|
environment:
|
|
internal
|
|
env:
|
|
PROMPT_FLOW_TEST_MODE: "live"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
# snok/install-poetry need this to support Windows
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.pythonVersion }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
|
- name: Azure Login
|
|
uses: azure/login@v1
|
|
with:
|
|
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
|
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
|
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
|
- name: Generate Configs
|
|
uses: "./.github/actions/step_generate_configs"
|
|
with:
|
|
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
|
- name: install promptflow-core from wheel
|
|
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
|
run: |
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
|
|
poetry run pip install -e ../promptflow-recording
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run core tests
|
|
run: poetry run pytest ./tests/core --tb=short
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: promptflow_core_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
|
|
promptflow_core_azureml_serving_tests:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
|
needs: build
|
|
env:
|
|
PROMPT_FLOW_TEST_MODE: "live"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
# snok/install-poetry need this to support Windows
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.pythonVersion }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
|
- name: install promptflow-core from wheel
|
|
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
|
run: |
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
|
poetry run pip install -e ../promptflow-recording
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run azureml-serving tests
|
|
run: poetry run pytest ./tests/azureml-serving --tb=short
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: promptflow_core_azureml_serving_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
|
|
|
|
promptflow_devkit_tests:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
|
needs: build
|
|
environment:
|
|
internal
|
|
env:
|
|
PROMPT_FLOW_TEST_MODE: "live"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
# snok/install-poetry need this to support Windows
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.pythonVersion }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
|
- name: Azure Login
|
|
uses: azure/login@v1
|
|
with:
|
|
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
|
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
|
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
|
- name: Generate Configs
|
|
uses: "./.github/actions/step_generate_configs"
|
|
with:
|
|
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
|
- name: install promptflow-devkit from wheel
|
|
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
|
run: |
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run devkit tests
|
|
run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest" --tb=short
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: promptflow_devkit_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
|
|
|
|
promptflow_azure_tests:
|
|
needs: build
|
|
environment:
|
|
internal
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
env:
|
|
PROMPT_FLOW_TEST_MODE: "live"
|
|
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
|
|
PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
|
|
# snok/install-poetry need this to support Windows
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.pythonVersion }}
|
|
- uses: snok/install-poetry@v1
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
|
- name: Azure Login
|
|
uses: azure/login@v1
|
|
with:
|
|
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
|
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
|
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
|
- name: Generate Configs
|
|
uses: "./.github/actions/step_generate_configs"
|
|
with:
|
|
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
|
- name: install promptflow-azure from wheel
|
|
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
|
run: |
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_azure-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
|
|
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
|
|
poetry run pip install -e ../promptflow-recording
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: install test dependency group
|
|
run: poetry install --only test
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: run azure tests
|
|
run: poetry run pytest ./tests/sdk_cli_azure_test -n auto -m "unittest or e2etest" --tb=short
|
|
working-directory: ${{ env.WORKING_DIRECTORY }}
|
|
- name: upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: promptflow_azure_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
|
path: |
|
|
${{ env.WORKING_DIRECTORY }}/*.xml
|
|
|
|
|
|
promptflow_executor_tests:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
|
needs: build
|
|
environment:
|
|
internal
|
|
env:
|
|
testWorkingDirectory: src/promptflow
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest, macos-13]
|
|
pythonVersion: ['3.9', '3.10', '3.11']
|
|
runs-on: ${{ matrix.os }}
|
|
steps:
|
|
- name: checkout
|
|
uses: actions/checkout@v4
|
|
- name: Display and Set Environment Variables
|
|
run:
|
|
env | sort >> $GITHUB_OUTPUT
|
|
shell: bash -el {0}
|
|
- name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
|
|
uses: "./.github/actions/step_create_python_environment"
|
|
with:
|
|
pythonVersion: ${{ matrix.pythonVersion }}
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: wheel
|
|
path: artifacts
|
|
- name: install recording
|
|
run: |
|
|
pip install vcrpy
|
|
pip install -e .
|
|
working-directory: ${{ env.RECORD_DIRECTORY }}
|
|
- name: Azure Login
|
|
uses: azure/login@v1
|
|
with:
|
|
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
|
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
|
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
|
- name: Generate Configs
|
|
uses: "./.github/actions/step_generate_configs"
|
|
with:
|
|
targetFolder: ${{ env.testWorkingDirectory }}
|
|
- name: Install pf
|
|
shell: pwsh
|
|
working-directory: artifacts
|
|
run: |
|
|
pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
|
|
pip uninstall -y promptflow-core promptflow-devkit promptflow-tracing
|
|
pip install ${{ github.workspace }}/src/promptflow-tracing
|
|
pip install ${{ github.workspace }}/src/promptflow-core
|
|
pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
|
|
pip install ${{ github.workspace }}/src/promptflow-azure
|
|
gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
|
|
gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
|
|
pip freeze
|
|
- name: Run Executor Unit Test
|
|
shell: pwsh
|
|
working-directory: ${{ github.workspace }}
|
|
run: |
|
|
pip install langchain
|
|
pip install numexpr
|
|
python scripts/building/run_coverage_tests.py `
|
|
-p ${{ github.workspace }}/src/promptflow/promptflow `
|
|
-t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
|
|
-l eastus `
|
|
-m "all" `
|
|
-o "${{ github.workspace }}/test-results-executor-unit.xml"
|
|
- name: Run Executor E2E Test
|
|
shell: pwsh
|
|
working-directory: ${{ github.workspace }}
|
|
run: |
|
|
pip install langchain langchain_community
|
|
pip install numexpr
|
|
python scripts/building/run_coverage_tests.py `
|
|
-p ${{ github.workspace }}/src/promptflow/promptflow `
|
|
-t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
|
|
-l eastus `
|
|
-m "all" `
|
|
-o "${{ github.workspace }}/test-results-executor-e2e.xml"
|
|
- name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
|
|
path: ${{ github.workspace }}/*.xml
|
|
|
|
|
|
publish-test-results:
|
|
name: "Publish Tests Results"
|
|
needs: [ promptflow_devkit_tests, promptflow_azure_tests, promptflow_executor_tests, promptflow_core_tests, promptflow_core_azureml_serving_tests ]
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
checks: write
|
|
pull-requests: write
|
|
contents: read
|
|
issues: read
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Download Artifacts
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
path: artifacts
|
|
- name: Publish Test Results
|
|
uses: EnricoMi/publish-unit-test-result-action@v2
|
|
with:
|
|
files: "artifacts/**/test-*.xml"
|