chore: import upstream snapshot with attribution
Flake8 Lint / flake8 (push) Waiting to run
Publish Promptflow Doc / Build (push) Waiting to run
Publish Promptflow Doc / Deploy (push) Blocked by required conditions
Spell check CI / Spell_Check (push) Waiting to run
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Flake8 Lint / flake8 (push) Waiting to run
Publish Promptflow Doc / Build (push) Waiting to run
Publish Promptflow Doc / Deploy (push) Blocked by required conditions
Spell check CI / Spell_Check (push) Waiting to run
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
This commit is contained in:
wehub-resource-sync
+277
@@ -0,0 +1,277 @@
|
||||
{
|
||||
"version": "0.2",
|
||||
"language": "en",
|
||||
"languageId": "python",
|
||||
"dictionaries": [
|
||||
"powershell",
|
||||
"python",
|
||||
"go",
|
||||
"css",
|
||||
"html",
|
||||
"bash",
|
||||
"npm",
|
||||
"softwareTerms",
|
||||
"en_us",
|
||||
"en-gb"
|
||||
],
|
||||
"ignorePaths": [
|
||||
"**/*.js",
|
||||
"**/*.mjs",
|
||||
"**/*.css",
|
||||
"**/*.pyc",
|
||||
"**/*.log",
|
||||
"**/*.jsonl",
|
||||
"**/*.xml",
|
||||
"**/*.txt",
|
||||
".gitignore",
|
||||
"examples/README.md",
|
||||
"examples/flex-flows/README.md",
|
||||
"examples/prompty/README.md",
|
||||
"scripts/docs/_build/**",
|
||||
"scripts/readme/**",
|
||||
"src/promptflow-azure/promptflow/azure/_restclient/flow/**",
|
||||
"src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
|
||||
"src/promptflow-azure/promptflow/azure/_models/**",
|
||||
"src/promptflow-azure/tests/**",
|
||||
"src/promptflow-core/promptflow/core/_connection_provider/_models/**",
|
||||
"src/promptflow/tests/**",
|
||||
"src/promptflow-devkit/tests/**",
|
||||
"src/promptflow-azure/tests/**",
|
||||
"src/promptflow-recording/**",
|
||||
"src/promptflow-tools/tests/**",
|
||||
"src/promptflow-devkit/promptflow/_sdk/_service/static/trace/index.html",
|
||||
"src/promptflow-devkit/promptflow/_sdk/_service/static/chat-window/index.html",
|
||||
"**/flow.dag.yaml",
|
||||
"**/pyproject.toml",
|
||||
"**/setup.py",
|
||||
"scripts/installer/curl_install_pypi/**",
|
||||
"scripts/installer/windows/**",
|
||||
".github/workflows/**",
|
||||
".github/actions/**",
|
||||
".github/pipelines/**",
|
||||
".github/CODEOWNERS",
|
||||
"src/promptflow-evals/tests/**",
|
||||
"benchmark/promptflow-serve/result-archive/**"
|
||||
],
|
||||
"words": [
|
||||
"amlignore",
|
||||
"aoai",
|
||||
"Apim",
|
||||
"astext",
|
||||
"attribited",
|
||||
"azureai",
|
||||
"azurecr",
|
||||
"azureml",
|
||||
"azuremlsdktestpypi",
|
||||
"fdnpromptflow",
|
||||
"mlsdkfdnprod",
|
||||
"Bhavik",
|
||||
"centralus",
|
||||
"chatml",
|
||||
"cref",
|
||||
"devui",
|
||||
"e2etest",
|
||||
"e2etests",
|
||||
"eastus",
|
||||
"Entra",
|
||||
"env",
|
||||
"faiss",
|
||||
"geval",
|
||||
"hnsw",
|
||||
"junit",
|
||||
"KHTML",
|
||||
"Likert",
|
||||
"llmlingua",
|
||||
"olleh",
|
||||
"logit",
|
||||
"logprobs",
|
||||
"meid",
|
||||
"mgmt",
|
||||
"MistralAI",
|
||||
"mldesigner",
|
||||
"mlflow",
|
||||
"msal",
|
||||
"msrest",
|
||||
"myconn",
|
||||
"numlines",
|
||||
"nunit",
|
||||
"openai",
|
||||
"pfazure",
|
||||
"pfbytes",
|
||||
"pfcli",
|
||||
"pfutil",
|
||||
"Policheck",
|
||||
"pydata",
|
||||
"Qdrant",
|
||||
"rediraffe",
|
||||
"retriable",
|
||||
"envsubst",
|
||||
"ROBOCOPY",
|
||||
"serp",
|
||||
"Summ",
|
||||
"tablefmt",
|
||||
"undoc",
|
||||
"UNLCK",
|
||||
"upia",
|
||||
"uvicorn",
|
||||
"vectordb",
|
||||
"vnet",
|
||||
"Weaviate",
|
||||
"westus",
|
||||
"wsid",
|
||||
"Xpia"
|
||||
],
|
||||
"ignoreWords": [
|
||||
"openmpi",
|
||||
"ipynb",
|
||||
"xdist",
|
||||
"pydash",
|
||||
"tqdm",
|
||||
"rtype",
|
||||
"epocs",
|
||||
"fout",
|
||||
"funcs",
|
||||
"todos",
|
||||
"fstring",
|
||||
"creds",
|
||||
"zipp",
|
||||
"gmtime",
|
||||
"pyjwt",
|
||||
"nbconvert",
|
||||
"nbformat",
|
||||
"pypandoc",
|
||||
"dotenv",
|
||||
"miniconda",
|
||||
"datas",
|
||||
"tcgetpgrp",
|
||||
"yamls",
|
||||
"fmt",
|
||||
"parameterised",
|
||||
"cicd",
|
||||
"sphagettification",
|
||||
"gaierror",
|
||||
"serpapi",
|
||||
"genutils",
|
||||
"metadatas",
|
||||
"tiktoken",
|
||||
"bfnrt",
|
||||
"orelse",
|
||||
"thead",
|
||||
"sympy",
|
||||
"ghactions",
|
||||
"esac",
|
||||
"MSRC",
|
||||
"pycln",
|
||||
"strictyaml",
|
||||
"psutil",
|
||||
"getch",
|
||||
"tcgetattr",
|
||||
"TCSADRAIN",
|
||||
"stringio",
|
||||
"jsonify",
|
||||
"werkzeug",
|
||||
"continuumio",
|
||||
"pydantic",
|
||||
"iterrows",
|
||||
"dtype",
|
||||
"fillna",
|
||||
"nlines",
|
||||
"aggr",
|
||||
"tcsetattr",
|
||||
"pysqlite",
|
||||
"AADSTS700082",
|
||||
"Pyinstaller",
|
||||
"runsvdir",
|
||||
"runsv",
|
||||
"levelno",
|
||||
"LANCZOS",
|
||||
"Mobius",
|
||||
"ruamel",
|
||||
"gunicorn",
|
||||
"pkill",
|
||||
"pgrep",
|
||||
"Hwfoxydrg",
|
||||
"llms",
|
||||
"vcrpy",
|
||||
"uionly",
|
||||
"llmops",
|
||||
"Abhishek",
|
||||
"restx",
|
||||
"httpx",
|
||||
"tiiuae",
|
||||
"nohup",
|
||||
"metagenai",
|
||||
"WBITS",
|
||||
"laddr",
|
||||
"nrows",
|
||||
"Dumpable",
|
||||
"XCLASS",
|
||||
"otel",
|
||||
"OTLP",
|
||||
"spawnv",
|
||||
"arxiv",
|
||||
"autogen",
|
||||
"spawnve",
|
||||
"addrs",
|
||||
"pycache",
|
||||
"pywin",
|
||||
"STARTF",
|
||||
"mltable",
|
||||
"prompty",
|
||||
"Prompty",
|
||||
"setenv",
|
||||
"cscript",
|
||||
"nologo",
|
||||
"wscript",
|
||||
"raisvc",
|
||||
"evals",
|
||||
"setenv",
|
||||
"pypdf",
|
||||
"redoc",
|
||||
"starlette",
|
||||
"mlindex",
|
||||
"redef",
|
||||
"rcts",
|
||||
"Chunker",
|
||||
"mpnet",
|
||||
"wargs",
|
||||
"dcid",
|
||||
"aiohttp",
|
||||
"endofprompt",
|
||||
"tkey",
|
||||
"tparam",
|
||||
"ncols",
|
||||
"piezo",
|
||||
"Piezo",
|
||||
"cmpop",
|
||||
"finalizer",
|
||||
"finalizers",
|
||||
"amlbi",
|
||||
"cmpop",
|
||||
"omap",
|
||||
"Machinal",
|
||||
"azureopenaimodelconfiguration",
|
||||
"openaimodelconfiguration",
|
||||
"usecwd",
|
||||
"upia",
|
||||
"xpia",
|
||||
"locustio",
|
||||
"euap",
|
||||
"Rerank",
|
||||
"rerank",
|
||||
"reranker",
|
||||
"rcfile",
|
||||
"pylintrc",
|
||||
"gleu",
|
||||
"Gleu",
|
||||
"GLEU",
|
||||
"fmeasure",
|
||||
"punkt",
|
||||
"BYOI",
|
||||
"Neur"
|
||||
],
|
||||
"flagWords": [
|
||||
"Prompt Flow"
|
||||
],
|
||||
"allowCompoundWords": true
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
FROM python:3.9-slim-bullseye AS base
|
||||
|
||||
RUN set -x
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get -y install curl \
|
||||
&& apt-get -y install net-tools \
|
||||
&& apt-get -y install procps \
|
||||
&& apt-get -y install build-essential \
|
||||
&& apt-get -y install docker.io
|
||||
|
||||
# Install notebook depenency
|
||||
RUN pip install ipython ipykernel
|
||||
RUN ipython kernel install --user --name promptflow
|
||||
|
||||
# FROM base AS promptflow
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
RUN set +x
|
||||
|
||||
CMD bash
|
||||
@@ -0,0 +1,13 @@
|
||||
# Devcontainer for promptflow
|
||||
To facilitate your promptflow project development and empower you to work on LLM projects using promptflow more effectively,
|
||||
we've configured the necessary environment for developing promptflow projects and utilizing flows through the dev container feature.
|
||||
You can seamlessly initiate your promptflow project development and start leveraging flows by simply using the dev container feature via VS Code or Codespaces.
|
||||
|
||||
## Use Github Codespaces
|
||||
Use codespaces to open promptflow repo, it will automatically build the dev containers environment and open promptflow with dev containers. You can just click: [](https://codespaces.new/microsoft/promptflow?quickstart=1)
|
||||
|
||||
## Use local devcontainer
|
||||
Use vscode to open promptflow repo, and install vscode extension: Dev Containers and then open promptflow with dev containers.
|
||||
|
||||
**About dev containers please refer to: [dev containers](https://code.visualstudio.com/docs/devcontainers/containers)**
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "Promptflow-Python39",
|
||||
// "context" is the path that the Codespaces docker build command should be run from, relative to devcontainer.json
|
||||
"context": ".",
|
||||
"dockerFile": "Dockerfile",
|
||||
"runArgs": ["-v", "/var/run/docker.sock:/var/run/docker.sock"],
|
||||
"remoteEnv": {
|
||||
"HOST_PROJECT_PATH": "${localWorkspaceFolder}"
|
||||
},
|
||||
"customizations": {
|
||||
"codespaces": {
|
||||
"openFiles": ["README.md", "examples/README.md"]
|
||||
},
|
||||
"vscode": {
|
||||
// Set *default* container specific settings.json values on container create.
|
||||
"settings": {
|
||||
"terminal.integrated.shell.linux": "/bin/bash"
|
||||
},
|
||||
// Add the IDs of extensions you want installed when the container is created.
|
||||
"extensions": [
|
||||
"ms-python.python",
|
||||
"ms-toolsai.vscode-ai",
|
||||
"ms-toolsai.jupyter",
|
||||
"redhat.vscode-yaml",
|
||||
"prompt-flow.prompt-flow"
|
||||
]
|
||||
}
|
||||
},
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/azure-cli:1": {}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
@@ -0,0 +1,2 @@
|
||||
promptflow[azure]>=1.9.0
|
||||
promptflow-tools
|
||||
@@ -0,0 +1,52 @@
|
||||
# CAUTION: Order is important; the LAST matching pattern takes the most precedence.
|
||||
# So put outside folders first, and then inside folders.
|
||||
# Don't change the sequence of the regions.
|
||||
|
||||
* @microsoft/prompt-flow-approvers
|
||||
|
||||
# 1. Folders
|
||||
/src/promptflow/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-tracing/ @microsoft/promptflow-execution
|
||||
/src/promptflow-core/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-devkit/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-azure/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-recording/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-evals/ @microsoft/pf-eval-team-contributor
|
||||
/src/promptflow-parallel/ @microsoft/promptflow-sdk
|
||||
/src/promptflow-rag/ @microsoft/promptflow-rag
|
||||
|
||||
/scripts/docs/ @microsoft/promptflow-sdk
|
||||
/scripts/installer/ @microsoft/promptflow-sdk
|
||||
/scripts/json_schema/ @microsoft/promptflow-sdk
|
||||
/scripts/readme/ @microsoft/promptflow-sdk
|
||||
|
||||
/src/promptflow-tools/ @microsoft/promptflow-tools
|
||||
/scripts/tool/ @microsoft/promptflow-tools
|
||||
/examples/tools/ @microsoft/promptflow-tools
|
||||
|
||||
# guard structure changes to docsite
|
||||
/docs/how-to-guides/index.md @microsoft/prompt-flow-doc-approvers
|
||||
/docs/cloud/index.md @microsoft/prompt-flow-doc-approvers
|
||||
|
||||
# 2. Subfolders
|
||||
/src/promptflow-tracing/promptflow/tracing/ @microsoft/promptflow-execution
|
||||
/src/promptflow-core/promptflow/_core/ @microsoft/promptflow-execution
|
||||
/src/promptflow-core/promptflow/executor/ @microsoft/promptflow-execution
|
||||
/src/promptflow-core/promptflow/integrations/ @microsoft/promptflow-execution
|
||||
/src/promptflow-core/promptflow/storage/ @microsoft/promptflow-execution
|
||||
/src/promptflow-devkit/promptflow/_sdk/_mlflow.py @brynn-code
|
||||
/src/promptflow-devkit/promptflow/_internal/ @microsoft/promptflow-execution
|
||||
/src/promptflow-devkit/promptflow/batch/ @microsoft/promptflow-execution
|
||||
/src/promptflow/tests/executor/ @microsoft/promptflow-execution
|
||||
|
||||
/src/promptflow-devkit/promptflow/_proxy/ @microsoft/promptflow-execution @microsoft/promptflow-sdk
|
||||
|
||||
/src/promptflow/tests/sdk_cli_test/ @microsoft/promptflow-sdk
|
||||
/src/promptflow/tests/sdk_cli_azure_test/ @microsoft/promptflow-sdk
|
||||
/src/promptflow/tests/sdk_cli_global_config_test/ @microsoft/promptflow-sdk
|
||||
/src/promptflow/tests/sdk_pfs_test/ @microsoft/promptflow-sdk
|
||||
/src/promptflow/tests/executor/_prompty_executor.py @microsoft/promptflow-execution @microsoft/promptflow-sdk
|
||||
/src/promptflow/tests/executor/_script_executor.py @microsoft/promptflow-execution @microsoft/promptflow-sdk
|
||||
/src/promptflow-tracing/promptflow/tracing/_start_trace.py @microsoft/promptflow-sdk
|
||||
|
||||
/docs/how-to-guides/develop-a-tool/ @microsoft/promptflow-tools
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: "[BUG]"
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of the bug.
|
||||
|
||||
**How To Reproduce the bug**
|
||||
Steps to reproduce the behavior, how frequent can you experience the bug:
|
||||
1.
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Running Information(please complete the following information):**
|
||||
- Promptflow Package Version using `pf -v`: [e.g. 0.0.102309906]
|
||||
- Operating System: [e.g. Ubuntu 20.04, Windows 11]
|
||||
- Python Version using `python --version`: [e.g. python==3.10.12]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
name: Contribution request
|
||||
about: Create a request of review to contribution
|
||||
title: "[Contribution Request]"
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your contribution request related to a problem? Please describe.**
|
||||
A clear and concise description of what you will contribute.
|
||||
|
||||
**Detail the functionality and value.**
|
||||
A clear and concise description of the contribution's functionality and value.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the contribution request here.
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: "[Feature Request]"
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is.
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
name: Bug report for Prompt flow VS Code extension
|
||||
about: Bug report for Prompt flow VS Code extension
|
||||
title: "[BUG] [VSCode Extension]"
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of the bug.
|
||||
|
||||
**How To Reproduce the bug**
|
||||
Steps to reproduce the behavior, how frequent can you experience the bug:
|
||||
1.
|
||||
|
||||
**Screenshots**
|
||||
1. On the VSCode primary side bar > the Prompt flow pane > quick access section. Find the "install dependencies" action. Please it and attach the screenshots there.
|
||||
2. Please provide other snapshots about the key steps to repro the issue.
|
||||
|
||||
**Environment Information**
|
||||
- Promptflow Package Version using `pf -v`: [e.g. 0.0.102309906]
|
||||
- Operating System: [e.g. Ubuntu 20.04, Windows 11]
|
||||
- Python Version using `python --version`: [e.g. python==3.10.12]
|
||||
- VS Code version.
|
||||
- Prompt Flow extension version.
|
||||
- On the VS Code bottom pane > Output pivot > "prompt flow" channel, find the error message could be relevant to the issue and past them here. That would be helpful for our trouble shooting.
|
||||
- If your code to repro the issue is public and you want to share us, please share the link.
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Description
|
||||
|
||||
Please add an informative description that covers that changes made by the pull request and link all relevant issues.
|
||||
|
||||
# All Promptflow Contribution checklist:
|
||||
- [ ] **The pull request does not introduce [breaking changes].**
|
||||
- [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.**
|
||||
- [ ] **I have read the [contribution guidelines](https://github.com/microsoft/promptflow/blob/main/CONTRIBUTING.md).**
|
||||
- [ ] **I confirm that all new dependencies are compatible with the MIT license.**
|
||||
- [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).**
|
||||
|
||||
## General Guidelines and Best Practices
|
||||
- [ ] Title of the pull request is clear and informative.
|
||||
- [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
|
||||
|
||||
### Testing Guidelines
|
||||
- [ ] Pull request includes test coverage for the included changes.
|
||||
@@ -0,0 +1,27 @@
|
||||
name: step_create_conda_environment
|
||||
inputs:
|
||||
condaEnvironmentFilePath:
|
||||
required: false
|
||||
default: "scripts/building/release-env.yml"
|
||||
type: string
|
||||
pythonVersion:
|
||||
required: false
|
||||
default: "3.9"
|
||||
type: string
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Setup Miniconda
|
||||
uses: conda-incubator/setup-miniconda@v2
|
||||
with:
|
||||
miniconda-version: "latest"
|
||||
activate-environment: release-env
|
||||
environment-file: ${{ inputs.condaEnvironmentFilePath }}
|
||||
python-version: ${{ inputs.pythonVersion }}
|
||||
auto-activate-base: false
|
||||
auto-update-conda: true
|
||||
- run: |
|
||||
conda info
|
||||
conda list
|
||||
python --version
|
||||
shell: bash -el {0}
|
||||
@@ -0,0 +1,25 @@
|
||||
name: step_create_python_environment
|
||||
inputs:
|
||||
pipFilePath:
|
||||
required: false
|
||||
default: "src/promptflow/dev_requirements.txt"
|
||||
type: string
|
||||
pythonVersion:
|
||||
required: false
|
||||
default: "3.9"
|
||||
type: string
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
with:
|
||||
version: 1.8.5
|
||||
- run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ inputs.pipFilePath }}
|
||||
pip freeze
|
||||
shell: pwsh
|
||||
@@ -0,0 +1,19 @@
|
||||
name: step_generate_configs
|
||||
inputs:
|
||||
targetFolder:
|
||||
required: false
|
||||
default: "."
|
||||
type: string
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Generate the connections config file
|
||||
working-directory: ${{ github.workspace }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
pip list
|
||||
pip install azure-identity
|
||||
pip install azure-keyvault
|
||||
echo "Generating connection config file..."
|
||||
python ./scripts/building/generate_connection_config.py `
|
||||
--target_folder ${{ inputs.targetFolder }}
|
||||
@@ -0,0 +1,11 @@
|
||||
name: step_merge_main
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Merge main to current branch
|
||||
working-directory: ${{ github.workspace }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
git config --global user.name 'prompt flow fundamental'
|
||||
git config --global user.email 'aml-pt-eng@microsoft.com'
|
||||
git pull --no-ff origin main
|
||||
@@ -0,0 +1,55 @@
|
||||
name: step_publish_test_results
|
||||
inputs:
|
||||
testActionFileName:
|
||||
required: required
|
||||
type: string
|
||||
testResultTitle:
|
||||
required: false
|
||||
default: "Test Result"
|
||||
type: string
|
||||
osVersion:
|
||||
required: false
|
||||
default: "ubuntu-latest"
|
||||
type: string
|
||||
pythonVersion:
|
||||
required: false
|
||||
default: "3.9"
|
||||
type: string
|
||||
coverageThreshold:
|
||||
required: false
|
||||
default: "0.3"
|
||||
type: string
|
||||
context:
|
||||
description: 'The context of the status'
|
||||
required: false
|
||||
default: 'test/sdk_cli'
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Display and Set Environment Variables
|
||||
run: env | sort >> $GITHUB_OUTPUT
|
||||
shell: bash -el {0}
|
||||
id: display_env
|
||||
- name: Publish Test Results
|
||||
uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: "${{ inputs.testResultTitle }} [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/${{ inputs.testActionFileName }}?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)"
|
||||
comment_title: "${{ inputs.testResultTitle }} [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/${{ inputs.testActionFileName }}?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)"
|
||||
files: "artifacts/**/test-*.xml"
|
||||
- name: Code Coverage Summary
|
||||
if: ${{ inputs.coverageThreshold != 0 }}
|
||||
uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/Test Results (Python ${{ inputs.pythonVersion }}) (OS ${{ inputs.osVersion }})/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: true
|
||||
format: markdown
|
||||
hide_branch_rate: false
|
||||
hide_complexity: true
|
||||
indicators: true
|
||||
output: both
|
||||
thresholds: '${{ inputs.coverageThreshold }} 80'
|
||||
@@ -0,0 +1,54 @@
|
||||
name: step_sdk_setup
|
||||
inputs:
|
||||
scriptPath:
|
||||
required: false
|
||||
type: string
|
||||
setupType:
|
||||
required: false
|
||||
default: promptflow_with_extra
|
||||
type: string
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Clean up installed packages
|
||||
working-directory: ${{ inputs.scriptPath }}
|
||||
continue-on-error: true
|
||||
shell: pwsh
|
||||
run: |
|
||||
pip uninstall -y promptflow promptflow-sdk promptflow-tools
|
||||
- name: 'Build and install: promptflow-sdk'
|
||||
if: inputs.setupType == 'promptflow_dev'
|
||||
shell: pwsh
|
||||
run: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r ./dev_requirements.txt
|
||||
python ./setup.py bdist_wheel
|
||||
$package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}
|
||||
pip install $package.FullName
|
||||
echo "########### pip freeze ###########"
|
||||
pip freeze
|
||||
working-directory: ${{ inputs.scriptPath }}
|
||||
- name: 'Build and install: promptflow-tools'
|
||||
shell: pwsh
|
||||
run: |
|
||||
Set-PSDebug -Trace 2
|
||||
python ./setup.py bdist_wheel
|
||||
$package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}
|
||||
pip install $package.FullName
|
||||
echo "########### pip freeze (After) ###########"
|
||||
pip freeze
|
||||
working-directory: src/promptflow-tools
|
||||
- name: 'Build and install: promptflow with extra'
|
||||
if: inputs.setupType == 'promptflow_with_extra'
|
||||
shell: pwsh
|
||||
run: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r ./dev_requirements.txt
|
||||
echo "########### pip list (Before) ###########"
|
||||
pip list
|
||||
python ./setup.py bdist_wheel
|
||||
$package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}
|
||||
pip install --force-reinstall $($package.FullName + "[azure,executable,azureml-serving,executor-service]")
|
||||
echo "########### pip freeze (After) ###########"
|
||||
pip freeze
|
||||
working-directory: ${{ inputs.scriptPath }}
|
||||
@@ -0,0 +1,23 @@
|
||||
examples:
|
||||
- examples/**
|
||||
documentation:
|
||||
- docs/**
|
||||
promptflow-tracing:
|
||||
- src/promptflow-tracing/**
|
||||
promptflow-core:
|
||||
- src/promptflow-core/**
|
||||
promptflow-devkit:
|
||||
- src/promptflow-devkit/**
|
||||
promptflow-azure:
|
||||
- src/promptflow-azure/**
|
||||
promptflow-evals:
|
||||
- src/promptflow-evals/**
|
||||
promptflow-parallel:
|
||||
- src/promptflow-parallel/**
|
||||
promptflow:
|
||||
- src/promptflow/**
|
||||
promptflow-tools:
|
||||
- src/promptflow-tools/**
|
||||
fundamental:
|
||||
- scripts/**
|
||||
- .github/**
|
||||
@@ -0,0 +1,74 @@
|
||||
# Pipeline link: https://dev.azure.com/msdata/Vienna/_build?definitionId=26179&_a=summary
|
||||
parameters:
|
||||
- name: policyCulture
|
||||
displayName: "Policy Culture"
|
||||
type: string
|
||||
# The culture used to run policy check scan, can be region codes separated by comma, e.g. 'en-US,de-DE'
|
||||
default: 'en-US'
|
||||
|
||||
name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.r) # Configure run or build numbers
|
||||
|
||||
variables:
|
||||
- name: sourceLocation
|
||||
value: $(System.DefaultWorkingDirectory)
|
||||
|
||||
trigger:
|
||||
- main
|
||||
- releases/*
|
||||
|
||||
pool:
|
||||
name: promptflow-1ES-win
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
|
||||
- task: PowerShell@2
|
||||
inputs:
|
||||
targetType: inline
|
||||
script: |
|
||||
Remove-Item -Recurse -Force $(sourceLocation)/benchmark/promptflow-serve/result-archive
|
||||
Remove-Item -Path $(sourceLocation)/src/promptflow-azure/promptflow/azure/_restclient/flow_service_caller.py -Force
|
||||
displayName: 'Delete benchmark HTML files, and files to avoid PoliCheck scan'
|
||||
|
||||
# https://eng.ms/docs/microsoft-security/cloud-ecosystem-security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/sdl-azdo-extension/PoliCheck-build-task
|
||||
- task: PoliCheck@2
|
||||
inputs:
|
||||
targetType: 'F'
|
||||
targetArgument: '$(sourceLocation)'
|
||||
optionsPE: '1'
|
||||
optionsUEPATH: '$(sourceLocation)\scripts\compliance-check\user_exclusion.xml'
|
||||
result: '$(sourceLocation)\scripts\compliance-check\result.tsv'
|
||||
optionsXCLASS: 'Geopolitical'
|
||||
|
||||
- task: PowerShell@2
|
||||
inputs:
|
||||
targetType: inline
|
||||
script: |
|
||||
if (-Not (Test-Path "$(sourceLocation)/scripts/compliance-check/result.tsv")) {
|
||||
Write-Host "PoliCheck@2 (previous step) is not supported for forked GitHub repository, which will break the following step."
|
||||
Write-Host "So as a workaround, this step will create an empty result.tsv file."
|
||||
New-Item -ItemType Directory -Force -Path "$(sourceLocation)/scripts/compliance-check"
|
||||
Set-Location "$(sourceLocation)/scripts/compliance-check"
|
||||
New-Item -ItemType File -Name "result.tsv"
|
||||
}
|
||||
displayName: 'Workaround for fork repository'
|
||||
|
||||
- task: PowerShell@2
|
||||
inputs:
|
||||
targetType: 'filePath'
|
||||
filePath: '$(sourceLocation)\scripts\compliance-check\Check-PolicheckScan.ps1'
|
||||
arguments: >
|
||||
-policheckResult $(sourceLocation)\scripts\compliance-check\result.tsv
|
||||
displayName: 'Check result'
|
||||
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: failed()
|
||||
inputs:
|
||||
targetPath: '$(sourceLocation)\scripts\compliance-check\result.tsv'
|
||||
artifactName: 'compliance-check-result'
|
||||
publishLocation: 'pipeline'
|
||||
|
||||
- task: CredScan@3
|
||||
displayName: 'CredScan'
|
||||
inputs:
|
||||
scanFolder: '$(sourceLocation)'
|
||||
@@ -0,0 +1,55 @@
|
||||
parameters:
|
||||
- name: promptflowCsPat
|
||||
displayName: "PAT to clone csharp repository"
|
||||
type: string
|
||||
- name: flowProjectRelativePath
|
||||
displayName: "Flow Project Relative Path"
|
||||
type: string
|
||||
|
||||
steps:
|
||||
- task: UseDotNet@2
|
||||
inputs:
|
||||
version: '6.x'
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.9.x'
|
||||
architecture: 'x64'
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Install promptflow cli'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r src/promptflow/dev_requirements.txt
|
||||
pip install src/promptflow-tracing
|
||||
pip install src/promptflow-core[executor-service]
|
||||
pip install src/promptflow-devkit
|
||||
pip install src/promptflow-azure
|
||||
pip install src/promptflow-recording
|
||||
pip freeze
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Clone csharp repository'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
git clone https://$(PROMPTFLOW_CS_PAT)@dev.azure.com/msdata/Vienna/_git/PromptflowCS csharp
|
||||
|
||||
- task: NuGetAuthenticate@1
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
inputs:
|
||||
command: 'restore'
|
||||
projects: '$(flowProjectRelativePath)/**/*.csproj'
|
||||
feedsToUse: 'config'
|
||||
nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
|
||||
displayName: 'dotnet restore'
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
inputs:
|
||||
command: 'build'
|
||||
projects: '$(flowProjectRelativePath)/**/*.csproj'
|
||||
feedsToUse: 'config'
|
||||
nugetConfigPath: '$(flowProjectRelativePath)/nuget.config'
|
||||
displayName: 'dotnet build'
|
||||
@@ -0,0 +1,46 @@
|
||||
parameters:
|
||||
- name: azureOpenAiApiKey
|
||||
displayName: "Azure OpenAI API Key"
|
||||
type: string
|
||||
- name: azureOpenAiApiBase
|
||||
displayName: "Azure OpenAI API Base"
|
||||
type: string
|
||||
- name: flowProjectRelativePath
|
||||
displayName: "Flow Project Relative Path"
|
||||
type: string
|
||||
|
||||
steps:
|
||||
- task: PowerShell@2
|
||||
displayName: 'Copy local connections for ci pipeline'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
Copy-Item dev-connections.json.example connections.json
|
||||
workingDirectory: $(Build.SourcesDirectory)/src/promptflow
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Run sdk cli tests'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
pytest tests/ -m "csharp"
|
||||
workingDirectory: $(Build.SourcesDirectory)/src/promptflow-devkit
|
||||
env:
|
||||
CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
|
||||
AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
|
||||
AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
|
||||
IS_IN_CI_PIPELINE: true
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Run azure sdk cli tests'
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: |
|
||||
pytest tests/sdk_cli_azure_test/e2etests/test_csharp_sdk.py
|
||||
workingDirectory: $(Build.SourcesDirectory)/src/promptflow-azure
|
||||
env:
|
||||
CSHARP_TEST_PROJECTS_ROOT: $(Build.SourcesDirectory)/$(flowProjectRelativePath)
|
||||
AZURE_OPENAI_API_KEY: $(azureOpenAiApiKey)
|
||||
AZURE_OPENAI_ENDPOINT: $(azureOpenAiApiBase)
|
||||
PROMPT_FLOW_TEST_MODE: "replay"
|
||||
IS_IN_CI_PIPELINE: true
|
||||
@@ -0,0 +1,100 @@
|
||||
# https://msdata.visualstudio.com/Vienna/_build?definitionId=33952&_a=summary
|
||||
name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.r) # Configure run or build numbers
|
||||
|
||||
variables:
|
||||
- group: promptflow-csharp
|
||||
- name: BuildConfiguration
|
||||
value: 'Debug'
|
||||
- name: currentRepoPath
|
||||
value: "csharp"
|
||||
- name: flowProjectRelativePath
|
||||
value: '$(currentRepoPath)/src/TestProjects'
|
||||
- name: system.debug
|
||||
value: 'true'
|
||||
|
||||
schedules:
|
||||
- cron: "40 18 * * *" # Every day starting at 2:40 BJT
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- releases/*
|
||||
paths:
|
||||
include:
|
||||
- src/promptflow-core/**
|
||||
- src/promptflow-devkit/**
|
||||
- src/promptflow/**
|
||||
- src/promptflow-tracing/**
|
||||
- scripts/building/**
|
||||
- .github/pipelines/promptflow-csharp-e2e-test.yml
|
||||
- src/promptflow-recording/**
|
||||
|
||||
parameters:
|
||||
- name: githubPromptflowBranch
|
||||
displayName: "Github Promptflow Branch Name"
|
||||
type: string
|
||||
default: main
|
||||
|
||||
jobs:
|
||||
- job: linux
|
||||
pool:
|
||||
name: promptflow-1ES-ubuntu20
|
||||
steps:
|
||||
- task: Bash@3
|
||||
displayName: 'Set environment variables'
|
||||
inputs:
|
||||
targetType: inline
|
||||
script: |
|
||||
export AOAI_CONNECTION=$(AOAI_CONNECTION)
|
||||
export OPENAI_CONNECTION=$(OPENAI_CONNECTION)
|
||||
export SERP_CONNECTION=$(SERP_CONNECTION)
|
||||
export ACS_CONNECTION=$(ACS_CONNECTION)
|
||||
export IS_IN_CI_PIPELINE=true
|
||||
|
||||
- template: promptflow-csharp-e2e-test-env-setup.yml
|
||||
parameters:
|
||||
flowProjectRelativePath: '$(flowProjectRelativePath)'
|
||||
promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
|
||||
|
||||
- template: promptflow-csharp-e2e-test-tests.yml
|
||||
parameters:
|
||||
flowProjectRelativePath: '$(flowProjectRelativePath)'
|
||||
azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
|
||||
azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
|
||||
|
||||
- publish: $(flowProjectRelativePath)
|
||||
condition: always()
|
||||
artifact: 'BuiltFlows-linux'
|
||||
- job: windows
|
||||
pool:
|
||||
name: promptflow-1ES-win
|
||||
steps:
|
||||
- task: PowerShell@2
|
||||
displayName: 'Set environment variables'
|
||||
inputs:
|
||||
targetType: inline
|
||||
script: |
|
||||
setx AOAI_CONNECTION $(AOAI_CONNECTION)
|
||||
setx OPENAI_CONNECTION $(OPENAI_CONNECTION)
|
||||
setx SERP_CONNECTION $(SERP_CONNECTION)
|
||||
setx ACS_CONNECTION $(ACS_CONNECTION)
|
||||
setx IS_IN_CI_PIPELINE true
|
||||
|
||||
- template: promptflow-csharp-e2e-test-env-setup.yml
|
||||
parameters:
|
||||
flowProjectRelativePath: '$(flowProjectRelativePath)'
|
||||
promptflowCsPat: '$(PROMPTFLOW_CS_PAT)'
|
||||
|
||||
- template: promptflow-csharp-e2e-test-tests.yml
|
||||
parameters:
|
||||
flowProjectRelativePath: '$(flowProjectRelativePath)'
|
||||
azureOpenAiApiBase: '$(AZURE_OPENAI_API_BASE)'
|
||||
azureOpenAiApiKey: '$(AZURE_OPENAI_API_KEY)'
|
||||
|
||||
- publish: $(flowProjectRelativePath)
|
||||
condition: always()
|
||||
artifact: 'BuiltFlows-windows'
|
||||
@@ -0,0 +1,336 @@
|
||||
---
|
||||
name: maf-online-endpoint
|
||||
description: "Deploy a Microsoft Agent Framework (MAF) workflow as a managed online endpoint to an Azure ML workspace or an Azure AI Foundry hub-based project. Wraps any workflow into an init()/run() scoring script, creates conda environment, endpoint and deployment YAMLs, deploy script, and assigns RBAC. Supports managed identity auth and Application Insights tracing. WHEN: deploy MAF workflow, deploy agent-framework workflow, create online endpoint for MAF, deploy workflow to AML, deploy workflow to Foundry project, managed online endpoint for agent workflow, wrap workflow in scoring script, deploy agent as endpoint, realtime endpoint in Foundry project."
|
||||
---
|
||||
|
||||
# Deploy MAF Workflow as a Managed Online Endpoint
|
||||
|
||||
This skill wraps a Microsoft Agent Framework (`agent-framework`) workflow into
|
||||
a managed online endpoint using the standard scoring-script pattern
|
||||
(`init()` / `run()`), following the patterns from the
|
||||
[azureml-examples managed endpoint samples](https://github.com/Azure/azureml-examples/tree/main/sdk/python/endpoints/online/managed).
|
||||
|
||||
The endpoint can be deployed to either:
|
||||
|
||||
| Deployment Target | Description |
|
||||
|-------------------|-------------|
|
||||
| **Azure Machine Learning workspace** | Standalone AML workspace — user provides subscription, resource group, and workspace name |
|
||||
| **Azure AI Foundry hub-based project** | An AI project living under a Foundry hub — the project name is the workspace name for `az ml` commands |
|
||||
|
||||
Both targets produce the **same generated files** (`online-deployment/` directory)
|
||||
and use the **same `az ml` CLI / `azure-ai-ml` Python SDK**. The difference is
|
||||
in how the workspace is identified and RBAC scope.
|
||||
|
||||
## Overview
|
||||
|
||||
The deployment creates all files in an `online-deployment/` subdirectory under
|
||||
the project root:
|
||||
|
||||
```
|
||||
<project-root>/
|
||||
workflow.py ← the MAF workflow
|
||||
online-deployment/
|
||||
score.py ← scoring script
|
||||
conda.yml ← conda environment
|
||||
endpoint.yml ← endpoint config
|
||||
deployment.yml ← deployment template (${VAR} placeholders)
|
||||
deploy.sh ← deploy script (Bash; see notes for Windows)
|
||||
.gitignore ← ignores rendered YAML with secrets
|
||||
```
|
||||
|
||||
1. **Scoring script** (`score.py`) — `init()` imports the workflow factory;
|
||||
`run()` creates a fresh workflow instance per request to avoid concurrency errors.
|
||||
2. **Conda environment** (`conda.yml`) — Python 3.11 with agent-framework and
|
||||
azureml-inference-server-http.
|
||||
3. **Endpoint YAML** (`endpoint.yml`) — endpoint name and auth mode.
|
||||
4. **Deployment YAML** (`deployment.yml`) — template with `${VAR}` placeholders
|
||||
for environment variables, instance config, and request settings.
|
||||
5. **Deploy script** (`deploy.sh`) — renders the template, creates the endpoint
|
||||
and deployment, runs a smoke test.
|
||||
|
||||
> **Path resolution rule:** AML CLI resolves `conda_file`, `code`, and
|
||||
> `scoring_script` paths **relative to the YAML file location**, not the CWD.
|
||||
> Since the YAML is inside `online-deployment/`, use `conda_file: conda.yml`
|
||||
> (same directory) and `code: ..` (parent = project root).
|
||||
|
||||
## Agent Interaction Pattern
|
||||
|
||||
When the user asks to deploy a MAF workflow as an online endpoint:
|
||||
|
||||
1. **Ask** for the **deployment target** using `vscode_askQuestions`:
|
||||
- **Azure Machine Learning workspace** — standalone AML workspace
|
||||
- **Azure AI Foundry project** — hub-based project (the project name is the
|
||||
workspace name)
|
||||
2. **Ask** for infrastructure variables (Step 0 §A) using `vscode_askQuestions`:
|
||||
subscription, resource group, workspace/project name, and the workflow file
|
||||
path.
|
||||
3. **Read** the workflow file. Inspect imports and `os.environ`/`os.getenv`
|
||||
calls to discover what the workflow needs (Step 0 §B).
|
||||
4. **Ask** the user to provide values for any workflow-specific variables that
|
||||
have no defaults.
|
||||
5. **Generate** the files from the templates in [./assets/](./assets/) into an
|
||||
`online-deployment/` subdirectory under the project root.
|
||||
6. **Run** deployment commands via terminal. On Windows, run `az` CLI commands
|
||||
directly in PowerShell (the Bash `deploy.sh` won't work). On Linux/macOS,
|
||||
use `deploy.sh` or run the commands directly.
|
||||
7. **Assign RBAC** (Step 6) — only needed for managed-identity workflows
|
||||
(Foundry/DefaultAzureCredential). Skip for API-key workflows. For Foundry
|
||||
hub-based projects, scope the role assignment to the hub's AI Services
|
||||
resource.
|
||||
8. **Wait** 5–10 minutes for RBAC propagation (if applicable), then run smoke
|
||||
test.
|
||||
9. **Report** the scoring URI and remind user to `.gitignore` rendered YAML
|
||||
files that contain secrets.
|
||||
|
||||
## Step 0 — Gather Required Information
|
||||
|
||||
### A. Online Endpoint Infrastructure (always required)
|
||||
|
||||
The same variables apply to both deployment targets. For a **Foundry hub-based
|
||||
project**, `WORKSPACE_NAME` is the **AI project name** (not the hub name).
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `SUBSCRIPTION_ID` | Azure subscription | _(required)_ |
|
||||
| `RESOURCE_GROUP` | Resource group containing the AML workspace or AI project | _(required)_ |
|
||||
| `WORKSPACE_NAME` | AML workspace name **or** AI Foundry project name | _(required)_ |
|
||||
| `ENDPOINT_NAME` | Name of the online endpoint | `maf-endpoint` |
|
||||
| `DEPLOYMENT_NAME` | Deployment name under the endpoint | `blue` |
|
||||
| `INSTANCE_TYPE` | VM SKU | `Standard_DS3_v2` |
|
||||
| `INSTANCE_COUNT` | Number of instances | `1` |
|
||||
| `REQUEST_TIMEOUT_MS` | Request timeout in ms | `60000` |
|
||||
|
||||
> **Foundry project note:** An AI Foundry hub-based project is backed by an AML
|
||||
> workspace. All `az ml` commands and the `MLClient` SDK work the same way —
|
||||
> just use the project name as `--workspace-name`. The endpoint scoring URI
|
||||
> format is identical: `https://<endpoint-name>.<region>.inference.ml.azure.com/score`
|
||||
|
||||
### B. Workflow Requirements (depends on the workflow)
|
||||
|
||||
Read the user's workflow file and inspect:
|
||||
|
||||
1. **Imports** — determine pip packages for `conda.yml`.
|
||||
2. **`os.environ[...]` / `os.getenv(...)` calls** — determine environment
|
||||
variables the deployment must inject.
|
||||
3. **Credential usage** — `DefaultAzureCredential` / `ManagedIdentityCredential`
|
||||
means RBAC must be set up; an API key means a secret env var.
|
||||
|
||||
#### Common workflow patterns
|
||||
|
||||
| Pattern | Imports | Required env vars | Extra pip packages | RBAC role |
|
||||
|---------|---------|--------------------|--------------------|-----------|
|
||||
| **Foundry LLM** | `FoundryChatClient`, `DefaultAzureCredential` | `FOUNDRY_PROJECT_ENDPOINT`, `FOUNDRY_MODEL` | `agent-framework` | `Cognitive Services User` |
|
||||
| **OpenAI API key** | `OpenAIChatClient` | `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT`, `AZURE_OPENAI_API_KEY` | `agent-framework`, `agent-framework-openai` | _(none — uses API key)_ |
|
||||
| **RAG (AI Search)** | `AzureAISearchContextProvider` | above + `AZURE_AI_SEARCH_ENDPOINT`, `AZURE_AI_SEARCH_INDEX_NAME`, `AZURE_AI_SEARCH_API_KEY` | above + `agent-framework-azure-ai-search` | above (Search uses API key) |
|
||||
| **Function tools** | plain Python functions | same as Foundry LLM | same as Foundry LLM | same as Foundry LLM |
|
||||
|
||||
### C. RBAC (after endpoint is created)
|
||||
|
||||
Get endpoint managed identity principal ID:
|
||||
```bash
|
||||
az ml online-endpoint show --name <endpoint> --query identity.principal_id -o tsv
|
||||
```
|
||||
|
||||
### D. Optional (cross-cutting)
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `APPLICATIONINSIGHTS_CONNECTION_STRING` | _(empty)_ | Enables OpenTelemetry tracing |
|
||||
|
||||
## Step 1 — Generate the Scoring Script
|
||||
|
||||
Use the template at [./assets/score.py](./assets/score.py).
|
||||
|
||||
**Key decisions:**
|
||||
- `AgentResponse` is not JSON-serializable → extract `.text` before returning.
|
||||
- `project_root = Path(__file__).resolve().parents[1]` — `score.py` is one
|
||||
level deep (`online-deployment/score.py`), so `parents[1]` reaches the
|
||||
project root. Adjust if your layout differs.
|
||||
- `asyncio.get_event_loop().run_until_complete()` bridges the sync `run()` to
|
||||
the async workflow.
|
||||
- Optional Application Insights tracing configured via env var.
|
||||
- **Input key:** Inspect the workflow to determine what key to parse from the
|
||||
request body (e.g. `"text"`, `"question"`). Adapt accordingly.
|
||||
- **Factory import:** `init()` imports the `create_workflow` factory from
|
||||
`workflow.py`. Each `run()` call invokes the factory to get a fresh workflow
|
||||
instance, avoiding `RuntimeError: Workflow is already running` on concurrent
|
||||
requests.
|
||||
|
||||
## Step 2 — Generate `conda.yml`
|
||||
|
||||
Use the template at [./assets/conda.yml](./assets/conda.yml).
|
||||
|
||||
**Important:**
|
||||
- Do NOT pin version constraints unless the user specifies them. Packages like
|
||||
`agent-framework-azure-ai-search` may not have published version ranges on
|
||||
PyPI, which causes image build failures.
|
||||
- Only include packages the workflow actually uses. For OpenAI API key
|
||||
workflows, include `agent-framework-openai` but omit
|
||||
`agent-framework-azure-ai-search` and `azure-monitor-opentelemetry` unless
|
||||
needed.
|
||||
|
||||
## Step 3 — Generate `endpoint.yml`
|
||||
|
||||
Use the template at [./assets/endpoint.yml](./assets/endpoint.yml).
|
||||
|
||||
## Step 4 — Generate `deployment.yml` (Template)
|
||||
|
||||
Use the template at [./assets/deployment.yml](./assets/deployment.yml).
|
||||
|
||||
**Critical — path resolution:**
|
||||
AML CLI resolves all relative paths in the deployment YAML **relative to the
|
||||
YAML file's location**, not the working directory. Since deployment files live
|
||||
in `online-deployment/`:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
conda_file: conda.yml # ← same dir as deployment.yml
|
||||
code_configuration:
|
||||
code: .. # ← parent dir = project root
|
||||
scoring_script: online-deployment/score.py # ← relative to code root
|
||||
```
|
||||
|
||||
Getting this wrong causes a double-nesting error like
|
||||
`online-deployment/online-deployment/conda.yml`.
|
||||
|
||||
**Other key settings:**
|
||||
- `request_timeout_ms: 60000` — LLM calls typically take 5–30 s; the AML
|
||||
default of 5 s causes timeouts.
|
||||
- Use `conda_file` (not `pip_requirements`) — the latter is not valid for
|
||||
inline environment definitions.
|
||||
- When rendering with `envsubst`, use a **restricted variable list** so
|
||||
`$schema` is not eaten.
|
||||
- Only include env vars the workflow actually needs; omit unused ones.
|
||||
|
||||
**Security:** The rendered YAML (`deployment-rendered.yml`) may contain API
|
||||
keys in plaintext. A `.gitignore` file is generated automatically to exclude
|
||||
it (see Step 4b).
|
||||
|
||||
## Step 4b — Generate `.gitignore`
|
||||
|
||||
Always create `online-deployment/.gitignore` to prevent rendered YAML files
|
||||
containing secrets from being committed:
|
||||
|
||||
```gitignore
|
||||
deployment-rendered.yml
|
||||
```
|
||||
|
||||
## Step 5 — Deploy
|
||||
|
||||
### Option A: Bash script (Linux/macOS)
|
||||
|
||||
Use the template at [./assets/deploy.sh](./assets/deploy.sh). Requires
|
||||
`envsubst` (part of `gettext`).
|
||||
|
||||
### Option B: Direct CLI commands (Windows / any OS)
|
||||
|
||||
On Windows, `deploy.sh` won't work (`envsubst`, `mktemp`, process substitution
|
||||
are unavailable). Instead, run the steps directly in PowerShell:
|
||||
|
||||
```powershell
|
||||
# 1. Render deployment YAML (replace placeholders with actual values)
|
||||
$content = Get-Content online-deployment/deployment.yml -Raw
|
||||
$content = $content -replace '\$\{AZURE_OPENAI_ENDPOINT\}', $env:AZURE_OPENAI_ENDPOINT
|
||||
# ... repeat for each placeholder ...
|
||||
Set-Content -Path online-deployment/deployment-rendered.yml -Value $content
|
||||
|
||||
# 2. Create endpoint
|
||||
az ml online-endpoint create `
|
||||
--subscription $SUBSCRIPTION_ID `
|
||||
--resource-group $RESOURCE_GROUP `
|
||||
--workspace-name $WORKSPACE_NAME `
|
||||
--file online-deployment/endpoint.yml
|
||||
|
||||
# 3. Create deployment (run from the project root directory!)
|
||||
az ml online-deployment create `
|
||||
--subscription $SUBSCRIPTION_ID `
|
||||
--resource-group $RESOURCE_GROUP `
|
||||
--workspace-name $WORKSPACE_NAME `
|
||||
--file online-deployment/deployment-rendered.yml `
|
||||
--all-traffic
|
||||
|
||||
# 4. Smoke test
|
||||
Set-Content -Path online-deployment/request.json -Value '{"text": "Hello"}'
|
||||
az ml online-endpoint invoke `
|
||||
--subscription $SUBSCRIPTION_ID `
|
||||
--resource-group $RESOURCE_GROUP `
|
||||
--workspace-name $WORKSPACE_NAME `
|
||||
--name <ENDPOINT_NAME> `
|
||||
--request-file online-deployment/request.json
|
||||
```
|
||||
|
||||
> **Important:** Run the `az ml online-deployment create` command from the
|
||||
> **project root** directory, not from inside `online-deployment/`. The CLI
|
||||
> resolves `code: ..` relative to the YAML file, but the CWD also matters for
|
||||
> finding the YAML file itself.
|
||||
|
||||
## Step 6 — RBAC for Managed Identity
|
||||
|
||||
After the endpoint is created, its system-assigned managed identity needs
|
||||
the **`Cognitive Services User`** role on the Foundry resource.
|
||||
|
||||
### Finding the AI Services resource
|
||||
|
||||
- **Standalone AML workspace:** The user must provide the AI Services
|
||||
(Cognitive Services) resource name and resource group.
|
||||
- **Foundry hub-based project:** The AI Services resource is linked to the hub.
|
||||
Find it via the Azure Portal (Hub → Connected resources) or via CLI:
|
||||
```bash
|
||||
az ml workspace show \
|
||||
--name <project-name> \
|
||||
--resource-group <rg> \
|
||||
--query "associated_workspaces" -o table
|
||||
```
|
||||
|
||||
### Assign the role
|
||||
|
||||
```bash
|
||||
# Get principal ID
|
||||
PRINCIPAL_ID=$(az ml online-endpoint show \
|
||||
--subscription "$SUBSCRIPTION_ID" \
|
||||
--name <ENDPOINT_NAME> \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--query identity.principal_id -o tsv)
|
||||
|
||||
# Assign role
|
||||
az role assignment create \
|
||||
--assignee-object-id "$PRINCIPAL_ID" \
|
||||
--assignee-principal-type ServicePrincipal \
|
||||
--role "Cognitive Services User" \
|
||||
--scope "/subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.CognitiveServices/accounts/<account>"
|
||||
```
|
||||
|
||||
**Why `Cognitive Services User`?**
|
||||
- `Azure AI Developer` does **not** include `Microsoft.CognitiveServices/accounts/AIServices/agents/write`.
|
||||
- `Cognitive Services User` has the wildcard `Microsoft.CognitiveServices/*`.
|
||||
- Allow **5–10 minutes** for RBAC data plane propagation.
|
||||
|
||||
### Foundry hub-based project — additional considerations
|
||||
|
||||
- The hub's **managed network** may restrict outbound access. Ensure the
|
||||
endpoint can reach the AI Services resource and any external APIs the workflow
|
||||
calls. If the hub uses a private endpoint, no extra steps are needed for
|
||||
AI Services calls within the same VNet.
|
||||
- The user deploying must have the **Azure AI Developer** role (or equivalent)
|
||||
on the resource group to create endpoints and deployments in the project.
|
||||
|
||||
See [./references/managed-identity.md](./references/managed-identity.md) for full details.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---------|-------|-----|
|
||||
| `No such file: .../online-deployment/online-deployment/conda.yml` | Paths in deployment YAML resolved relative to YAML location, not CWD | Use `conda_file: conda.yml` and `code: ..` when YAML is in a subdirectory |
|
||||
| `401 PermissionDenied` | Missing RBAC | Assign `Cognitive Services User` on Foundry resource |
|
||||
| `upstream request timeout` | 5 s default too short | `request_timeout_ms: 60000` |
|
||||
| `AgentResponse is not JSON serializable` | Returning raw workflow output | Extract `.text` from the response |
|
||||
| `pip_requirements` validation error | Invalid field for inline env | Use `conda_file` instead |
|
||||
| Image build fails on version constraints | Package not on PyPI with that version | Remove version pins from `conda.yml` |
|
||||
| `$schema` missing after envsubst | Unrestricted envsubst eats `$schema` | Use restricted variable list |
|
||||
| `FileNotFoundError: az` (Windows subprocess) | `az` is a `.cmd` file on Windows | Use `shell=True` in `subprocess.run` |
|
||||
| `envsubst` not found (Windows) | `envsubst` is a Linux tool | Use PowerShell string replacement (see Step 5 Option B) |
|
||||
| Deployment fails in Foundry project with network error | Hub managed network blocks outbound access | Check hub network settings; add outbound rules for required endpoints |
|
||||
| Cannot create endpoint in Foundry project | Insufficient RBAC on the project | User needs `Azure AI Developer` role on the resource group |
|
||||
|
||||
See [./references/troubleshooting.md](./references/troubleshooting.md) for extended diagnostics.
|
||||
@@ -0,0 +1,13 @@
|
||||
name: maf-env
|
||||
channels:
|
||||
- defaults
|
||||
dependencies:
|
||||
- python=3.11
|
||||
- pip
|
||||
- pip:
|
||||
- agent-framework
|
||||
- agent-framework-azure-ai-search
|
||||
- python-dotenv
|
||||
- azure-monitor-opentelemetry
|
||||
- azure-identity
|
||||
- azureml-inference-server-http
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/bin/bash
|
||||
# Deploys a MAF workflow as an Azure ML Managed Online Endpoint.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - az login completed
|
||||
# - Azure ML workspace already exists
|
||||
# - The ml CLI extension is installed: az extension add -n ml
|
||||
# - envsubst is available (part of gettext)
|
||||
#
|
||||
# NOTE: This script requires Bash (Linux/macOS). On Windows, run the
|
||||
# az CLI commands directly in PowerShell — see SKILL.md Step 6, Option B.
|
||||
#
|
||||
# Required environment variables:
|
||||
# SUBSCRIPTION_ID - Azure subscription ID
|
||||
# RESOURCE_GROUP - Resource group containing the AML workspace
|
||||
# WORKSPACE_NAME - Azure ML workspace name
|
||||
#
|
||||
# Workflow-specific environment variables (uncomment/add as needed):
|
||||
# Foundry pattern: FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_MODEL
|
||||
# OpenAI pattern: AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_API_KEY
|
||||
#
|
||||
# Optional environment variables:
|
||||
# INSTANCE_TYPE - VM SKU (default: Standard_DS3_v2)
|
||||
# INSTANCE_COUNT - Number of instances (default: 1)
|
||||
# APPLICATIONINSIGHTS_CONNECTION_STRING - App Insights connection string
|
||||
#
|
||||
# Usage:
|
||||
# cd <project-root>
|
||||
# bash online-deployment/deploy.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd)
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
SUBSCRIPTION_ID="${SUBSCRIPTION_ID:?Set SUBSCRIPTION_ID}"
|
||||
RESOURCE_GROUP="${RESOURCE_GROUP:?Set RESOURCE_GROUP}"
|
||||
WORKSPACE_NAME="${WORKSPACE_NAME:?Set WORKSPACE_NAME}"
|
||||
|
||||
# ── Export variables that deployment.yml references via envsubst ──────
|
||||
export INSTANCE_TYPE="${INSTANCE_TYPE:-Standard_DS3_v2}"
|
||||
export INSTANCE_COUNT="${INSTANCE_COUNT:-1}"
|
||||
export APPLICATIONINSIGHTS_CONNECTION_STRING="${APPLICATIONINSIGHTS_CONNECTION_STRING:-}"
|
||||
# Add workflow-specific variables here, e.g.:
|
||||
# export AZURE_OPENAI_ENDPOINT="${AZURE_OPENAI_ENDPOINT:?Set AZURE_OPENAI_ENDPOINT}"
|
||||
# export AZURE_OPENAI_DEPLOYMENT="${AZURE_OPENAI_DEPLOYMENT:?Set AZURE_OPENAI_DEPLOYMENT}"
|
||||
# export AZURE_OPENAI_API_KEY="${AZURE_OPENAI_API_KEY:?Set AZURE_OPENAI_API_KEY}"
|
||||
|
||||
# ── Render deployment YAML (restricted envsubst preserves $schema) ───
|
||||
RENDERED_DEPLOYMENT="${SCRIPT_DIR}/deployment-rendered.yml"
|
||||
# List ONLY the variables your deployment.yml uses:
|
||||
SUBST_VARS='${INSTANCE_TYPE} ${INSTANCE_COUNT} ${APPLICATIONINSIGHTS_CONNECTION_STRING}'
|
||||
envsubst "$SUBST_VARS" < "${SCRIPT_DIR}/deployment.yml" > "$RENDERED_DEPLOYMENT"
|
||||
|
||||
echo "WARNING: ${RENDERED_DEPLOYMENT} may contain secrets. Add to .gitignore."
|
||||
|
||||
# ── Create managed online endpoint ──────────────────────────────────
|
||||
echo "Creating online endpoint..."
|
||||
az ml online-endpoint create \
|
||||
--subscription "$SUBSCRIPTION_ID" \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--file "${SCRIPT_DIR}/endpoint.yml" \
|
||||
2>/dev/null || echo "Endpoint already exists, continuing..."
|
||||
|
||||
# ── Create deployment under the endpoint ─────────────────────────────
|
||||
# IMPORTANT: Run from the project root so that `code: ..` in the YAML
|
||||
# resolves correctly (YAML is in online-deployment/, code root is parent).
|
||||
echo "Creating deployment (this takes 5-10 minutes)..."
|
||||
az ml online-deployment create \
|
||||
--subscription "$SUBSCRIPTION_ID" \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--file "$RENDERED_DEPLOYMENT" \
|
||||
--all-traffic
|
||||
|
||||
# ── Smoke test ───────────────────────────────────────────────────────
|
||||
ENDPOINT_NAME=$(grep '^name:' "${SCRIPT_DIR}/endpoint.yml" | awk '{print $2}')
|
||||
|
||||
echo "Running smoke test..."
|
||||
REQUEST_FILE=$(mktemp --suffix=.json)
|
||||
echo '{"text": "Hello World!"}' > "$REQUEST_FILE"
|
||||
az ml online-endpoint invoke \
|
||||
--subscription "$SUBSCRIPTION_ID" \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--name "$ENDPOINT_NAME" \
|
||||
--request-file "$REQUEST_FILE"
|
||||
rm -f "$REQUEST_FILE"
|
||||
|
||||
echo ""
|
||||
echo "Endpoint deployed successfully."
|
||||
SCORING_URI=$(az ml online-endpoint show \
|
||||
--subscription "$SUBSCRIPTION_ID" \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--name "$ENDPOINT_NAME" \
|
||||
--query "scoring_uri" -o tsv)
|
||||
echo "Scoring URI: ${SCORING_URI}"
|
||||
@@ -0,0 +1,26 @@
|
||||
$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json
|
||||
name: blue
|
||||
endpoint_name: maf-endpoint
|
||||
environment:
|
||||
name: maf-env
|
||||
version: "1"
|
||||
image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest
|
||||
conda_file: conda.yml
|
||||
code_configuration:
|
||||
code: ..
|
||||
scoring_script: online-deployment/score.py
|
||||
environment_variables:
|
||||
# --- Foundry / managed-identity pattern ---
|
||||
# FOUNDRY_PROJECT_ENDPOINT: ${FOUNDRY_PROJECT_ENDPOINT}
|
||||
# FOUNDRY_MODEL: ${FOUNDRY_MODEL}
|
||||
# --- OpenAI API-key pattern ---
|
||||
# AZURE_OPENAI_ENDPOINT: ${AZURE_OPENAI_ENDPOINT}
|
||||
# AZURE_OPENAI_DEPLOYMENT: ${AZURE_OPENAI_DEPLOYMENT}
|
||||
# AZURE_OPENAI_API_KEY: ${AZURE_OPENAI_API_KEY}
|
||||
# --- Optional ---
|
||||
APPLICATIONINSIGHTS_CONNECTION_STRING: ${APPLICATIONINSIGHTS_CONNECTION_STRING}
|
||||
instance_type: ${INSTANCE_TYPE}
|
||||
instance_count: ${INSTANCE_COUNT}
|
||||
request_settings:
|
||||
request_timeout_ms: 60000
|
||||
max_concurrent_requests_per_instance: 5
|
||||
@@ -0,0 +1,3 @@
|
||||
$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineEndpoint.schema.json
|
||||
name: maf-endpoint
|
||||
auth_mode: key
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Azure ML managed online endpoint scoring script for MAF workflows.
|
||||
|
||||
init() is called once when the container starts.
|
||||
run(raw_data) is called for each request; raw_data is the JSON request body.
|
||||
|
||||
This file lives in online-deployment/ under the project root.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_create_workflow = None
|
||||
|
||||
|
||||
def init():
|
||||
"""Called once when the endpoint container starts."""
|
||||
global _create_workflow
|
||||
|
||||
# Add the project root to sys.path so workflow modules can be imported.
|
||||
# score.py is in online-deployment/ (1 level deep), so parents[1] = project root.
|
||||
project_root = Path(__file__).resolve().parents[1]
|
||||
if str(project_root) not in sys.path:
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
# Optional Application Insights tracing.
|
||||
appinsights_conn = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
|
||||
if appinsights_conn:
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
configure_azure_monitor(connection_string=appinsights_conn)
|
||||
configure_otel_providers()
|
||||
|
||||
# Import the workflow factory. Every run() call creates a fresh workflow
|
||||
# instance to avoid "Workflow is already running" errors on concurrent
|
||||
# requests.
|
||||
from workflow import create_workflow
|
||||
|
||||
_create_workflow = create_workflow
|
||||
logger.info("Workflow factory loaded successfully.")
|
||||
|
||||
|
||||
def run(raw_data):
|
||||
"""Called for each scoring request.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_data : str
|
||||
JSON string with the request body, e.g. '{"text": "..."}'
|
||||
Adapt the input key to match what the workflow expects.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
{"answer": str}
|
||||
"""
|
||||
data = json.loads(raw_data)
|
||||
# Adapt input key to match the workflow's expected input.
|
||||
# Common keys: "text", "question", "input", "query"
|
||||
text = data.get("text", "").strip()
|
||||
if not text:
|
||||
raise ValueError("'text' field must not be empty.")
|
||||
|
||||
workflow = _create_workflow()
|
||||
result = asyncio.get_event_loop().run_until_complete(workflow.run(text))
|
||||
outputs = result.get_outputs()
|
||||
|
||||
if not outputs:
|
||||
raise RuntimeError("Workflow produced no output.")
|
||||
|
||||
output = outputs[0]
|
||||
# AgentResponse objects are not JSON-serializable; extract the text.
|
||||
if hasattr(output, "text"):
|
||||
return {"answer": output.text}
|
||||
return {"answer": str(output)}
|
||||
@@ -0,0 +1,105 @@
|
||||
# Using Managed Identity Instead of API Keys
|
||||
|
||||
For production deployments, use managed identity with `FoundryChatClient`.
|
||||
This removes the need to store API keys in your online endpoint environment
|
||||
variables entirely.
|
||||
|
||||
Azure ML managed online endpoints automatically have a **system-assigned
|
||||
managed identity** — no extra step is needed to enable it.
|
||||
|
||||
This pattern follows the
|
||||
[azureml-examples managed identity sample](https://github.com/Azure/azureml-examples/blob/main/sdk/python/endpoints/online/managed/managed-identities/online-endpoints-managed-identity-sai.ipynb).
|
||||
|
||||
## Step 1: Get the endpoint's managed identity principal ID
|
||||
|
||||
```bash
|
||||
az ml online-endpoint show \
|
||||
--name maf-endpoint \
|
||||
--resource-group <your-rg> \
|
||||
--workspace-name <your-workspace> \
|
||||
--query identity.principal_id -o tsv
|
||||
```
|
||||
|
||||
## Step 2: Grant the identity access to the Foundry project
|
||||
|
||||
```bash
|
||||
az role assignment create \
|
||||
--role "Cognitive Services User" \
|
||||
--assignee-object-id <principal-id-from-step-1> \
|
||||
--assignee-principal-type ServicePrincipal \
|
||||
--scope /subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.CognitiveServices/accounts/<foundry-resource>
|
||||
```
|
||||
|
||||
> **Note:** Use `Cognitive Services User` (not `Azure AI Developer`).
|
||||
> The `Azure AI Developer` role does not include the
|
||||
> `Microsoft.CognitiveServices/accounts/AIServices/agents/write` data action
|
||||
> required by Foundry. Allow 5–10 minutes for RBAC data plane propagation.
|
||||
|
||||
## Step 3: Update your client code
|
||||
|
||||
Use `DefaultAzureCredential()` — it automatically selects managed identity
|
||||
when running in Azure and Azure CLI credentials locally:
|
||||
|
||||
```python
|
||||
from agent_framework import Agent
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from azure.identity import DefaultAzureCredential
|
||||
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=DefaultAzureCredential(),
|
||||
)
|
||||
agent = Agent(client=client, name="MyAgent", instructions="...")
|
||||
```
|
||||
|
||||
For explicit managed identity in production-only code:
|
||||
|
||||
```python
|
||||
from azure.identity import ManagedIdentityCredential
|
||||
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=ManagedIdentityCredential(),
|
||||
)
|
||||
```
|
||||
|
||||
## Python SDK — Programmatic RBAC Assignment
|
||||
|
||||
Following the managed identity sample, you can assign roles programmatically:
|
||||
|
||||
```python
|
||||
from azure.mgmt.authorization import AuthorizationManagementClient
|
||||
from azure.mgmt.authorization.models import RoleAssignmentCreateParameters
|
||||
from azure.identity import AzureCliCredential
|
||||
import uuid
|
||||
|
||||
credential = AzureCliCredential()
|
||||
|
||||
# Get endpoint identity
|
||||
endpoint = ml_client.online_endpoints.get("maf-endpoint")
|
||||
principal_id = endpoint.identity.principal_id
|
||||
|
||||
# Create authorization client
|
||||
auth_client = AuthorizationManagementClient(
|
||||
credential=credential,
|
||||
subscription_id=subscription_id,
|
||||
api_version="2020-10-01-preview",
|
||||
)
|
||||
|
||||
# Find Cognitive Services User role
|
||||
scope = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.CognitiveServices/accounts/{foundry_account}"
|
||||
role_defs = auth_client.role_definitions.list(scope=scope)
|
||||
role_def = next(r for r in role_defs if r.role_name == "Cognitive Services User")
|
||||
|
||||
# Assign
|
||||
auth_client.role_assignments.create(
|
||||
scope=scope,
|
||||
role_assignment_name=str(uuid.uuid4()),
|
||||
parameters=RoleAssignmentCreateParameters(
|
||||
role_definition_id=role_def.id,
|
||||
principal_id=principal_id,
|
||||
),
|
||||
)
|
||||
```
|
||||
@@ -0,0 +1,174 @@
|
||||
# Troubleshooting MAF Online Endpoint Deployments
|
||||
|
||||
## Common Issues
|
||||
|
||||
### 1. Path double-nesting: `online-deployment/online-deployment/conda.yml`
|
||||
|
||||
**Cause:** AML CLI resolves `conda_file`, `code`, and `scoring_script` paths
|
||||
**relative to the YAML file's location**, not the working directory. If the
|
||||
deployment YAML is in `online-deployment/` and specifies
|
||||
`conda_file: online-deployment/conda.yml`, it resolves to
|
||||
`online-deployment/online-deployment/conda.yml`.
|
||||
|
||||
**Fix:** Use paths relative to the YAML file's directory:
|
||||
```yaml
|
||||
# deployment.yml is in online-deployment/
|
||||
environment:
|
||||
conda_file: conda.yml # same dir as deployment.yml
|
||||
code_configuration:
|
||||
code: .. # parent dir = project root
|
||||
scoring_script: online-deployment/score.py # relative to code root
|
||||
```
|
||||
|
||||
### 2. `401 PermissionDenied` / `AIServices/agents/write`
|
||||
|
||||
**Cause:** The endpoint's managed identity does not have the correct RBAC role
|
||||
on the Foundry resource.
|
||||
|
||||
**Fix:**
|
||||
```bash
|
||||
# Get endpoint identity
|
||||
PRINCIPAL_ID=$(az ml online-endpoint show \
|
||||
--name maf-endpoint \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--query identity.principal_id -o tsv)
|
||||
|
||||
# Assign Cognitive Services User (not Azure AI Developer)
|
||||
az role assignment create \
|
||||
--assignee-object-id "$PRINCIPAL_ID" \
|
||||
--assignee-principal-type ServicePrincipal \
|
||||
--role "Cognitive Services User" \
|
||||
--scope "/subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.CognitiveServices/accounts/<account>"
|
||||
```
|
||||
|
||||
Allow **5–10 minutes** for RBAC data plane propagation.
|
||||
|
||||
### 3. `upstream request timeout`
|
||||
|
||||
**Cause:** The default AML request timeout is 5 seconds, which is too short
|
||||
for LLM calls (typically 5–30 s).
|
||||
|
||||
**Fix:** Set `request_timeout_ms: 60000` in `deployment.yml`:
|
||||
```yaml
|
||||
request_settings:
|
||||
request_timeout_ms: 60000
|
||||
```
|
||||
|
||||
### 4. `AgentResponse is not JSON serializable`
|
||||
|
||||
**Cause:** The `run()` function returns a raw `AgentResponse` object.
|
||||
|
||||
**Fix:** Extract `.text` before returning:
|
||||
```python
|
||||
output = outputs[0]
|
||||
if hasattr(output, "text"):
|
||||
return {"answer": output.text}
|
||||
return {"answer": str(output)}
|
||||
```
|
||||
|
||||
### 5. `pip_requirements` validation error
|
||||
|
||||
**Cause:** `pip_requirements` is not a valid field for inline environment
|
||||
definitions in AML deployment YAML.
|
||||
|
||||
**Fix:** Use `conda_file` with a full conda environment definition:
|
||||
```yaml
|
||||
environment:
|
||||
name: maf-env
|
||||
version: "1"
|
||||
image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest
|
||||
conda_file: conda.yml
|
||||
```
|
||||
|
||||
### 6. Image build fails on package version
|
||||
|
||||
**Cause:** A pinned version constraint references a version that doesn't exist
|
||||
on PyPI for that package.
|
||||
|
||||
**Fix:** Remove version constraints from `conda.yml`:
|
||||
```yaml
|
||||
# Bad
|
||||
- agent-framework-azure-ai-search==1.0.0
|
||||
|
||||
# Good
|
||||
- agent-framework-azure-ai-search
|
||||
```
|
||||
|
||||
### 7. `$schema` missing after envsubst
|
||||
|
||||
**Cause:** Unrestricted `envsubst` treats `$schema` as a variable and replaces
|
||||
it with an empty string.
|
||||
|
||||
**Fix:** Use a restricted variable list:
|
||||
```bash
|
||||
SUBST_VARS='${FOUNDRY_PROJECT_ENDPOINT} ${FOUNDRY_MODEL} ...'
|
||||
envsubst "$SUBST_VARS" < deployment.yml > deployment-rendered.yml
|
||||
```
|
||||
|
||||
### 8. Workflow import errors at container startup
|
||||
|
||||
**Cause:** The `sys.path` depth in `score.py` doesn't match the project layout,
|
||||
so the workflow module cannot be imported.
|
||||
|
||||
**Fix:** Adjust the `parents[N]` depth in `score.py`:
|
||||
```python
|
||||
# If score.py is at project/deployment/score.py (1 level deep)
|
||||
project_root = Path(__file__).resolve().parents[1]
|
||||
|
||||
# If score.py is at project/phase-4/deployment/score.py (2 levels deep)
|
||||
project_root = Path(__file__).resolve().parents[2]
|
||||
```
|
||||
|
||||
### 9. Windows-specific issues
|
||||
|
||||
**`FileNotFoundError: az` in subprocess:**
|
||||
On Windows, `az` is a batch file (`az.cmd`), not a direct executable. Use
|
||||
`shell=True` in `subprocess.run()` calls.
|
||||
|
||||
**`envsubst` not found:**
|
||||
`envsubst` is a Linux tool (part of `gettext`). On Windows, use PowerShell
|
||||
string replacement instead:
|
||||
```powershell
|
||||
$content = Get-Content deployment.yml -Raw
|
||||
$content = $content -replace '\$\{VAR_NAME\}', $actualValue
|
||||
Set-Content -Path deployment-rendered.yml -Value $content
|
||||
```
|
||||
|
||||
**Process substitution `<(...)` not available:**
|
||||
Write request payloads to a temp file instead:
|
||||
```powershell
|
||||
Set-Content -Path request.json -Value '{"text": "Hello"}'
|
||||
az ml online-endpoint invoke ... --request-file request.json
|
||||
```
|
||||
|
||||
## Checking Deployment Logs
|
||||
|
||||
```bash
|
||||
# Get deployment logs
|
||||
az ml online-deployment get-logs \
|
||||
--name blue \
|
||||
--endpoint-name maf-endpoint \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--lines 100
|
||||
```
|
||||
|
||||
Or via Python SDK:
|
||||
```python
|
||||
ml_client.online_deployments.get_logs(
|
||||
name="blue",
|
||||
endpoint_name="maf-endpoint",
|
||||
lines=100,
|
||||
)
|
||||
```
|
||||
|
||||
## Checking Endpoint Health
|
||||
|
||||
```bash
|
||||
az ml online-endpoint show \
|
||||
--name maf-endpoint \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--workspace-name "$WORKSPACE_NAME" \
|
||||
--query "{name:name, state:provisioning_state, scoring_uri:scoring_uri, traffic:traffic}"
|
||||
```
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Deploy a MAF workflow as an Azure ML managed online endpoint using the Python SDK.
|
||||
|
||||
This script follows the pattern from:
|
||||
https://github.com/Azure/azureml-examples/tree/main/sdk/python/endpoints/online/managed
|
||||
|
||||
Usage:
|
||||
Set environment variables, then run:
|
||||
python deploy_sdk.py
|
||||
|
||||
Required env vars:
|
||||
SUBSCRIPTION_ID, RESOURCE_GROUP, WORKSPACE_NAME
|
||||
|
||||
Optional env vars:
|
||||
ENDPOINT_NAME (default: maf-endpoint)
|
||||
DEPLOYMENT_NAME (default: blue)
|
||||
INSTANCE_TYPE (default: Standard_DS3_v2)
|
||||
INSTANCE_COUNT (default: 1)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from azure.ai.ml import MLClient
|
||||
from azure.ai.ml.entities import (
|
||||
ManagedOnlineEndpoint,
|
||||
ManagedOnlineDeployment,
|
||||
Environment,
|
||||
CodeConfiguration,
|
||||
)
|
||||
from azure.identity import DefaultAzureCredential
|
||||
|
||||
|
||||
def main():
|
||||
subscription_id = os.environ["SUBSCRIPTION_ID"]
|
||||
resource_group = os.environ["RESOURCE_GROUP"]
|
||||
workspace_name = os.environ["WORKSPACE_NAME"]
|
||||
|
||||
endpoint_name = os.getenv("ENDPOINT_NAME", "maf-endpoint")
|
||||
deployment_name = os.getenv("DEPLOYMENT_NAME", "blue")
|
||||
instance_type = os.getenv("INSTANCE_TYPE", "Standard_DS3_v2")
|
||||
instance_count = int(os.getenv("INSTANCE_COUNT", "1"))
|
||||
|
||||
# Resolve paths
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
assets_dir = script_dir.parent / "assets"
|
||||
# Project root where code_configuration.code points to.
|
||||
# Adjust for your layout — this should be the directory containing workflow.py.
|
||||
project_root = script_dir.parents[3]
|
||||
|
||||
credential = DefaultAzureCredential()
|
||||
ml_client = MLClient(credential, subscription_id, resource_group, workspace_name)
|
||||
|
||||
# --- Create endpoint ---
|
||||
print(f"Creating endpoint '{endpoint_name}'...")
|
||||
endpoint = ManagedOnlineEndpoint(
|
||||
name=endpoint_name,
|
||||
description="MAF workflow online endpoint",
|
||||
auth_mode="key",
|
||||
)
|
||||
ml_client.online_endpoints.begin_create_or_update(endpoint).result()
|
||||
print(f"Endpoint '{endpoint_name}' ready.")
|
||||
|
||||
# --- Show managed identity ---
|
||||
endpoint = ml_client.online_endpoints.get(endpoint_name)
|
||||
print(f"Endpoint identity type: {endpoint.identity.type}")
|
||||
print(f"Endpoint identity principal_id: {endpoint.identity.principal_id}")
|
||||
|
||||
# --- Create deployment ---
|
||||
print(f"Creating deployment '{deployment_name}'...")
|
||||
env = Environment(
|
||||
name="maf-env",
|
||||
version="1",
|
||||
conda_file=str(assets_dir / "conda.yml"),
|
||||
image="mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest",
|
||||
)
|
||||
|
||||
# Add workflow-specific environment variables here, e.g.:
|
||||
# Foundry pattern: FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_MODEL
|
||||
# OpenAI pattern: AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_API_KEY
|
||||
env_vars = {}
|
||||
|
||||
deployment = ManagedOnlineDeployment(
|
||||
name=deployment_name,
|
||||
endpoint_name=endpoint_name,
|
||||
environment=env,
|
||||
code_configuration=CodeConfiguration(
|
||||
code=str(project_root),
|
||||
scoring_script=str(assets_dir / "score.py"),
|
||||
),
|
||||
instance_type=instance_type,
|
||||
instance_count=instance_count,
|
||||
environment_variables=env_vars,
|
||||
request_settings={"request_timeout_ms": 60000, "max_concurrent_requests_per_instance": 5},
|
||||
)
|
||||
ml_client.online_deployments.begin_create_or_update(deployment).result()
|
||||
print(f"Deployment '{deployment_name}' ready.")
|
||||
|
||||
# --- Set traffic ---
|
||||
endpoint.traffic = {deployment_name: 100}
|
||||
ml_client.online_endpoints.begin_create_or_update(endpoint).result()
|
||||
print(f"Traffic set to 100% on '{deployment_name}'.")
|
||||
|
||||
# --- Smoke test ---
|
||||
print("Running smoke test...")
|
||||
result = ml_client.online_endpoints.invoke(
|
||||
endpoint_name=endpoint_name,
|
||||
deployment_name=deployment_name,
|
||||
request_file=None,
|
||||
request_body=json.dumps({"text": "Hello World!"}),
|
||||
)
|
||||
print(f"Smoke test result: {result}")
|
||||
|
||||
# --- Report ---
|
||||
endpoint = ml_client.online_endpoints.get(endpoint_name)
|
||||
print(f"\nScoring URI: {endpoint.scoring_uri}")
|
||||
print(f"Swagger URI: {endpoint.swagger_uri}")
|
||||
print(f"Principal ID: {endpoint.identity.principal_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Tests for the MAF online endpoint skill assets.
|
||||
|
||||
Validates:
|
||||
- score.py can be imported and has init/run functions
|
||||
- YAML templates are valid YAML
|
||||
- deploy.sh is syntactically valid bash
|
||||
- conda.yml has required packages
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ASSETS_DIR = Path(__file__).resolve().parent.parent / "assets"
|
||||
|
||||
|
||||
class TestScoreScript:
|
||||
"""Validate the scoring script template."""
|
||||
|
||||
def _load_module(self):
|
||||
spec = importlib.util.spec_from_file_location("score", ASSETS_DIR / "score.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
# Don't exec — just check structure
|
||||
return spec, module
|
||||
|
||||
def test_score_file_exists(self):
|
||||
assert (ASSETS_DIR / "score.py").exists()
|
||||
|
||||
def test_has_init_function(self):
|
||||
content = (ASSETS_DIR / "score.py").read_text()
|
||||
assert "def init():" in content
|
||||
|
||||
def test_has_run_function(self):
|
||||
content = (ASSETS_DIR / "score.py").read_text()
|
||||
assert "def run(raw_data):" in content
|
||||
|
||||
def test_handles_agent_response(self):
|
||||
"""score.py must extract .text from AgentResponse."""
|
||||
content = (ASSETS_DIR / "score.py").read_text()
|
||||
assert 'hasattr(output, "text")' in content
|
||||
|
||||
def test_handles_empty_question(self):
|
||||
content = (ASSETS_DIR / "score.py").read_text()
|
||||
assert "must not be empty" in content
|
||||
|
||||
|
||||
class TestCondaYml:
|
||||
"""Validate the conda environment template."""
|
||||
|
||||
def test_file_exists(self):
|
||||
assert (ASSETS_DIR / "conda.yml").exists()
|
||||
|
||||
def test_valid_yaml(self):
|
||||
with open(ASSETS_DIR / "conda.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert isinstance(data, dict)
|
||||
|
||||
def test_has_python_311(self):
|
||||
with open(ASSETS_DIR / "conda.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
deps = data.get("dependencies", [])
|
||||
assert "python=3.11" in deps
|
||||
|
||||
def test_has_required_pip_packages(self):
|
||||
with open(ASSETS_DIR / "conda.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
pip_deps = None
|
||||
for dep in data.get("dependencies", []):
|
||||
if isinstance(dep, dict) and "pip" in dep:
|
||||
pip_deps = dep["pip"]
|
||||
break
|
||||
assert pip_deps is not None
|
||||
assert "agent-framework" in pip_deps
|
||||
assert "azureml-inference-server-http" in pip_deps
|
||||
assert "azure-identity" in pip_deps
|
||||
|
||||
def test_no_version_pins(self):
|
||||
"""Verify no strict version pins that could break builds."""
|
||||
with open(ASSETS_DIR / "conda.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
for dep in data.get("dependencies", []):
|
||||
if isinstance(dep, dict) and "pip" in dep:
|
||||
for pkg in dep["pip"]:
|
||||
assert "==" not in pkg, f"Version pin found: {pkg}"
|
||||
|
||||
|
||||
class TestEndpointYml:
|
||||
"""Validate the endpoint YAML template."""
|
||||
|
||||
def test_file_exists(self):
|
||||
assert (ASSETS_DIR / "endpoint.yml").exists()
|
||||
|
||||
def test_valid_yaml(self):
|
||||
with open(ASSETS_DIR / "endpoint.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert isinstance(data, dict)
|
||||
|
||||
def test_has_schema(self):
|
||||
with open(ASSETS_DIR / "endpoint.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert "$schema" in data
|
||||
|
||||
def test_has_auth_mode(self):
|
||||
with open(ASSETS_DIR / "endpoint.yml") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert data.get("auth_mode") == "key"
|
||||
|
||||
|
||||
class TestDeploymentYml:
|
||||
"""Validate the deployment YAML template."""
|
||||
|
||||
def test_file_exists(self):
|
||||
assert (ASSETS_DIR / "deployment.yml").exists()
|
||||
|
||||
def test_has_schema_line(self):
|
||||
content = (ASSETS_DIR / "deployment.yml").read_text()
|
||||
assert "$schema:" in content
|
||||
|
||||
def test_has_request_timeout(self):
|
||||
content = (ASSETS_DIR / "deployment.yml").read_text()
|
||||
assert "request_timeout_ms: 60000" in content
|
||||
|
||||
def test_has_conda_file_not_pip_requirements(self):
|
||||
content = (ASSETS_DIR / "deployment.yml").read_text()
|
||||
assert "conda_file:" in content
|
||||
assert "pip_requirements:" not in content
|
||||
|
||||
def test_uses_envsubst_placeholders(self):
|
||||
content = (ASSETS_DIR / "deployment.yml").read_text()
|
||||
assert "${FOUNDRY_PROJECT_ENDPOINT}" in content
|
||||
assert "${FOUNDRY_MODEL}" in content
|
||||
|
||||
def test_environment_variables_section(self):
|
||||
content = (ASSETS_DIR / "deployment.yml").read_text()
|
||||
assert "environment_variables:" in content
|
||||
|
||||
|
||||
class TestDeployScript:
|
||||
"""Validate the deploy shell script."""
|
||||
|
||||
def test_file_exists(self):
|
||||
assert (ASSETS_DIR / "deploy.sh").exists()
|
||||
|
||||
def test_uses_restricted_envsubst(self):
|
||||
"""envsubst must use restricted vars to preserve $schema."""
|
||||
content = (ASSETS_DIR / "deploy.sh").read_text()
|
||||
assert "SUBST_VARS=" in content
|
||||
assert 'envsubst "$SUBST_VARS"' in content
|
||||
|
||||
def test_has_set_euo_pipefail(self):
|
||||
content = (ASSETS_DIR / "deploy.sh").read_text()
|
||||
assert "set -euo pipefail" in content
|
||||
|
||||
def test_requires_mandatory_vars(self):
|
||||
content = (ASSETS_DIR / "deploy.sh").read_text()
|
||||
assert "SUBSCRIPTION_ID" in content
|
||||
assert "RESOURCE_GROUP" in content
|
||||
assert "WORKSPACE_NAME" in content
|
||||
assert "FOUNDRY_PROJECT_ENDPOINT" in content
|
||||
assert "FOUNDRY_MODEL" in content
|
||||
|
||||
def test_has_smoke_test(self):
|
||||
content = (ASSETS_DIR / "deploy.sh").read_text()
|
||||
assert "smoke test" in content.lower()
|
||||
|
||||
|
||||
class TestSkillMd:
|
||||
"""Validate SKILL.md structure."""
|
||||
|
||||
def test_skill_md_exists(self):
|
||||
skill_md = Path(__file__).resolve().parent.parent / "SKILL.md"
|
||||
assert skill_md.exists()
|
||||
|
||||
def test_has_frontmatter(self):
|
||||
skill_md = Path(__file__).resolve().parent.parent / "SKILL.md"
|
||||
content = skill_md.read_text(encoding="utf-8")
|
||||
assert content.startswith("---")
|
||||
# Should have closing frontmatter
|
||||
assert content.count("---") >= 2
|
||||
|
||||
def test_frontmatter_has_name(self):
|
||||
skill_md = Path(__file__).resolve().parent.parent / "SKILL.md"
|
||||
content = skill_md.read_text(encoding="utf-8")
|
||||
assert "name: maf-online-endpoint" in content
|
||||
|
||||
def test_frontmatter_has_description(self):
|
||||
skill_md = Path(__file__).resolve().parent.parent / "SKILL.md"
|
||||
content = skill_md.read_text(encoding="utf-8")
|
||||
assert "description:" in content
|
||||
|
||||
|
||||
class TestReferences:
|
||||
"""Validate reference documents exist."""
|
||||
|
||||
REFS_DIR = Path(__file__).resolve().parent.parent / "references"
|
||||
|
||||
def test_managed_identity_md(self):
|
||||
assert (self.REFS_DIR / "managed-identity.md").exists()
|
||||
|
||||
def test_safe_rollout_md(self):
|
||||
assert (self.REFS_DIR / "safe-rollout.md").exists()
|
||||
|
||||
def test_troubleshooting_md(self):
|
||||
assert (self.REFS_DIR / "troubleshooting.md").exists()
|
||||
@@ -0,0 +1,250 @@
|
||||
---
|
||||
name: maf-prs-job
|
||||
description: "Convert an existing Prompt Flow Parallel Run Step (PRS) pipeline submission into an Azure ML PRS pipeline that runs a Microsoft Agent Framework (MAF) workflow. Wraps the MAF workflow into a PRS init()/run() entry script, generates the parallel component YAML and conda environment, and rewrites the pipeline submission script. Replaces what `load_component(flow.dag.yaml)` did automatically for Prompt Flow \u2014 produces the hand-built equivalent so that downstream pipeline code (`flow_node = flow_component(...)`, `flow_node.outputs.flow_outputs`, `flow_node.outputs.debug_info`, `flow_node.mini_batch_size`, scheduler, batch endpoint) stays unchanged. WHEN: convert promptflow PRS to MAF PRS, migrate PRS pipeline to agent framework, wrap MAF workflow as parallel component, bulk run MAF workflow, run agent framework as parallel run step, batch run MAF workflow on AML, submit MAF workflow as pipeline component, replace flow.dag.yaml with MAF workflow in pipeline, load_component equivalent for MAF workflow, MAF version of flow_component, load MAF workflow as component, wrap MAF workflow as flow component, MAF flow component, replace flow_node in pipeline with MAF workflow, keep flow_outputs and debug_info ports with MAF, MAF parallel component with connections={}, run MAF workflow as flow_node in AML pipeline, load_component('workflow.py') doesn't work. DO NOT USE FOR: converting the flow itself (use promptflow-to-maf), deploying as online endpoint (use maf-online-endpoint), enabling tracing only (use maf-tracing)."
|
||||
license: MIT
|
||||
metadata:
|
||||
author: Team
|
||||
version: "0.1.0-draft"
|
||||
---
|
||||
|
||||
# Prompt Flow PRS → MAF PRS Pipeline Conversion
|
||||
|
||||
> Convert an existing **Prompt Flow Parallel Run Step (PRS)** pipeline submission
|
||||
> (one that uses `load_component("flow.dag.yaml")`) into a PRS pipeline that
|
||||
> runs a **Microsoft Agent Framework (MAF) workflow** as the parallel component.
|
||||
>
|
||||
> What `load_component(flow.dag.yaml)` did automatically — and which pieces
|
||||
> this skill produces by hand — is documented in
|
||||
> [references/pf-vs-maf-prs.md §0](references/pf-vs-maf-prs.md).
|
||||
|
||||
## Triggers
|
||||
|
||||
Activate this skill when the user wants to:
|
||||
|
||||
- Convert a Prompt Flow PRS pipeline submission to use a MAF workflow instead.
|
||||
- Run a MAF workflow as a parallel / bulk job on AML compute.
|
||||
- Replace `load_component(flow.dag.yaml)` with a hand-built parallel component
|
||||
that wraps a MAF workflow.
|
||||
|
||||
Also activate on these PF-user phrasings (people who learned from the
|
||||
[run-flow-with-pipeline notebook](../../../examples/tutorials/run-flow-with-pipeline/pipeline.ipynb)
|
||||
will describe their need in PF terms):
|
||||
|
||||
- "How do I `load_component` a MAF workflow?" / "`load_component('workflow.py')`
|
||||
doesn't work — what's the right way?"
|
||||
- "Give me the MAF equivalent of `flow_component = load_component('flow.dag.yaml')`."
|
||||
- "Wrap my MAF workflow as a **flow component** / **parallel component** /
|
||||
**PF-style component**."
|
||||
- "I want to use my MAF workflow as `flow_node` in my existing AML pipeline."
|
||||
- "My pipeline does `result_parser_node(pf_output_data=flow_node.outputs.flow_outputs,
|
||||
pf_debug_data=flow_node.outputs.debug_info)` — keep that working with MAF."
|
||||
- "How do I pass `connections={...}` / `${data.url}` column mapping when the
|
||||
step is a MAF workflow instead of a flow?"
|
||||
|
||||
In all of these cases the user's mental model is the PF auto-converted **flow
|
||||
component** with predefined `data` / `flow_outputs` / `debug_info` ports.
|
||||
This skill produces the hand-built MAF equivalent and **preserves those names**
|
||||
so downstream pipeline DSL, scheduler, and batch-endpoint code copy-paste
|
||||
unchanged.
|
||||
|
||||
Do **not** use this skill to convert the `flow.dag.yaml` itself — that is the
|
||||
job of [promptflow-to-maf](../promptflow-to-maf/SKILL.md). This skill assumes
|
||||
the MAF workflow already exists (or will be produced by `promptflow-to-maf`)
|
||||
and only deals with the **PRS / pipeline plumbing** around it.
|
||||
|
||||
---
|
||||
|
||||
## Outputs
|
||||
|
||||
For an input project containing a MAF workflow (`workflow.py` exporting
|
||||
`create_workflow()`) and an existing PF PRS submission script, **add the
|
||||
PRS plumbing into the MAF workflow folder itself** (default — keeps
|
||||
`workflow.py` and its deployment package together so customers manage one
|
||||
folder per workflow):
|
||||
|
||||
```
|
||||
<maf-workflow-folder>/
|
||||
├── workflow.py ← existing MAF workflow (untouched)
|
||||
├── requirements.txt ← existing (untouched)
|
||||
├── src/ ← ADDED: PRS entry + plumbing
|
||||
│ ├── entry.py ← thin PRS wrapper: init / run(mini_batch, context) / shutdown
|
||||
│ ├── hooks.py ← THE ONLY USER-EDITED FILE: setup / build_workflow_input / serialize_output
|
||||
│ └── maf_prs/ ← generic plumbing (mirrors promptflow-parallel's processor/executor split)
|
||||
│ ├── __init__.py
|
||||
│ ├── config.py ← argparse → MafPrsConfig
|
||||
│ ├── executor.py ← per-row driver; calls into hooks.py
|
||||
│ └── processor.py ← mini-batch dispatch, event-loop reuse, finalize
|
||||
├── component.yaml ← ADDED: Azure ML parallel component (replaces flow.dag.yaml)
|
||||
├── env/
|
||||
│ └── conda.yml ← ADDED: runtime env (agent-framework + AML PRS deps)
|
||||
├── submit_pipeline.py ← ADDED: MLClient + @pipeline DSL submission driver
|
||||
└── data/sample.jsonl ← ADDED only when the source `Input(path=...)` is a local file the agent can read; reused verbatim for cloud paths
|
||||
```
|
||||
|
||||
The **original PF flow folder is never modified** (it's a separate
|
||||
folder). Existing files in the MAF folder (`workflow.py`,
|
||||
`requirements.txt`, tests, …) are also left untouched — only **new**
|
||||
files are added next to them.
|
||||
|
||||
The only file the user normally needs to edit after generation is
|
||||
`src/hooks.py` — `build_workflow_input(row)`, `serialize_output(output)`,
|
||||
and the optional `setup(config)` — and even those are auto-filled when
|
||||
the source provides enough information (see
|
||||
[auto-derive-checks.md](references/auto-derive-checks.md)).
|
||||
`maf_prs/executor.py` and the rest of the package are **generic** and can
|
||||
be vendored unchanged across all converted workflows.
|
||||
|
||||
### Alternative: sibling folder layout (opt-in)
|
||||
|
||||
If the user explicitly asks to keep the MAF folder pristine (e.g. it is a
|
||||
public doc sample), generate a sibling folder named
|
||||
`<maf-workflow-folder>-prs/` instead, and **copy** `workflow.py` into it
|
||||
so `code: ./` in `component.yaml` ships it to AML. Trade-off: duplicate
|
||||
`workflow.py` to keep in sync. Default to the consolidated layout above
|
||||
unless asked.
|
||||
|
||||
---
|
||||
|
||||
## Core Rules
|
||||
|
||||
1. **Read both sides first.** Parse the user's PF PRS submission (script
|
||||
or notebook cells) and the MAF workflow (`workflow.py`, must export
|
||||
`create_workflow()`). If `create_workflow()` is missing, route the user
|
||||
to [promptflow-to-maf](../promptflow-to-maf/SKILL.md) first.
|
||||
2. **Auto-fill only when the source is unambiguous.** Run the checks in
|
||||
[auto-derive-checks.md](references/auto-derive-checks.md) and emit
|
||||
generated code only for fields that pass. For everything else leave a
|
||||
`# TODO` stub that quotes the original PF source and the missing piece —
|
||||
**never invent endpoint URLs, data paths, or untyped handler inputs**.
|
||||
3. **One workflow instance per row.** MAF workflows do not support
|
||||
concurrent `run()` on the same instance. The template
|
||||
`executor.execute(...)` builds a fresh workflow per row from the cached
|
||||
`_create_workflow` factory; do not "optimise" by caching an instance.
|
||||
4. **One asyncio loop per worker.** `processor.init()` creates the loop;
|
||||
`process()` reuses it via `run_until_complete`; `finalize()` closes it.
|
||||
Do not call `asyncio.run()` per row — it leaks Azure SDK transports.
|
||||
5. **Preserve PRS contract.** `entry.py` exposes exactly three top-level
|
||||
functions: `init()`, `run(mini_batch, context)`, `shutdown()`.
|
||||
`context.global_row_index_lower_bound` is required to stamp a stable
|
||||
`line_number` on each result; downstream PF eval tooling joins inputs
|
||||
to outputs by it.
|
||||
6. **Mirror PRS run settings 1:1.** Every PF PRS knob has an exact AML
|
||||
parallel-component equivalent; carry values across unchanged unless the
|
||||
user asks otherwise. See
|
||||
[pf-vs-maf-prs.md §4](references/pf-vs-maf-prs.md) for the table.
|
||||
7. **`connections=` → component inputs + Managed Identity.** Surface
|
||||
endpoint URL / deployment / API version as component `inputs:`, pass
|
||||
them via `program_arguments`. Prefer Managed Identity + Key Vault for
|
||||
secrets; never hard-code keys in `component.yaml`.
|
||||
8. **Generated project must be self-contained.** Whether using the
|
||||
default consolidated layout (PRS files added to the MAF folder) or the
|
||||
sibling-folder layout, no path should refer back to the original PF
|
||||
flow folder. Copy data samples, prompt files, and any user packages
|
||||
the workflow imports. In sibling-folder mode, also copy `workflow.py`
|
||||
so AML's `code:` snapshot ships it.
|
||||
|
||||
---
|
||||
|
||||
## Workflow
|
||||
|
||||
A single five-step loop. Each step combines the **decision** (what to
|
||||
ask / what to print to the user) with the **action** (what to write).
|
||||
|
||||
### 1. Ask
|
||||
|
||||
Use `vscode_askQuestions` for any of the following that are not obvious
|
||||
from the workspace:
|
||||
|
||||
- Path to the existing PF PRS submission (script or notebook cell).
|
||||
- Path to the MAF workflow (`workflow.py` with `create_workflow()`).
|
||||
- Whether the workflow has been migrated yet — if not, route to
|
||||
[promptflow-to-maf](../promptflow-to-maf/SKILL.md) first.
|
||||
|
||||
### 2. Audit
|
||||
|
||||
Extract the PRS settings from the source script using the table in
|
||||
[pf-vs-maf-prs.md §4](references/pf-vs-maf-prs.md) (compute,
|
||||
mini_batch_size, retry_settings, etc.) and **show the populated table to
|
||||
the user** before continuing.
|
||||
|
||||
### 3. Decide (Phase 1.5)
|
||||
|
||||
Run the checks in
|
||||
[auto-derive-checks.md](references/auto-derive-checks.md) (A–J) and
|
||||
**print the verdict table** showing which fields will be auto-filled vs.
|
||||
left as TODO. The same table doubles as the change log handed to the user
|
||||
in step 5.
|
||||
|
||||
### 4. Generate
|
||||
|
||||
Add `assets/` files **into the MAF workflow folder** (default) or into a
|
||||
new sibling `<maf-workflow-folder>-prs/` (only if the user opted in).
|
||||
Do not overwrite any pre-existing file in the MAF folder; if a file name
|
||||
already exists (e.g. `submit_pipeline.py`), confirm with the user before
|
||||
overwriting.
|
||||
|
||||
| File(s) | Action |
|
||||
|---|---|
|
||||
| `src/entry.py` | Copy verbatim. Do **not** edit. |
|
||||
| `src/maf_prs/{__init__,config,processor,executor}.py` | Copy verbatim. Do **not** edit unless the workflow needs an extra component input (then add a flag in `config.parse_args` and surface it in `component.yaml`). |
|
||||
| `src/hooks.py` | Apply auto-derived bodies for `build_workflow_input` / `serialize_output` per the verdict table; insert TODO stubs (template in [auto-derive-checks.md](references/auto-derive-checks.md)) where checks failed. If component inputs need to be turned into env vars / file paths before the workflow imports, fill the `setup(config)` body too. Add the matching `from workflow import ...` line at the top. |
|
||||
| `component.yaml` | Fill `inputs:` from check F; fill PRS settings from the audit table; set `program_arguments` to forward inputs + `--output_dir ${{outputs.debug_info}}`. Use `code: ./` and `entry_script: src/entry.py` so `workflow.py` (sibling of `src/`) is shipped to AML. **Set `data` input `type: uri_file`** and **always include the PF compatibility flag set** in `program_arguments` (`--amlbi_pf_enabled True --amlbi_pf_run_mode component --amlbi_file_format jsonl --amlbi_mini_batch_rows 1`) — PRS rejects bare `uri_file` without these flags (gotcha #12). **Wrap every `optional: true` input in `$[[--flag ${{inputs.X}}]]`** in `program_arguments` — bare `${{inputs.X}}` for an optional input fails registration with `Optional input X must be placed in nested argument: $[[]]` (gotcha #15). **Do not** add `--pf_input_*` flags. |
|
||||
| `env/conda.yml` | Add any extra pip packages imported by the workflow, **always with a lower-bound version pin** (`package>=X.Y.Z`) — bare entries trigger `error: resolution-too-deep` on the AML image build host and the job never reaches `init()` (gotcha #14). Keep the existing PRS runtime pins as-is. |
|
||||
| `submit_pipeline.py` | Fill `data_input` (check H) by **preserving the source `Input(path=..., type=..., mode=...)` verbatim** — same path, same type, same mode. Only rewrite the path to `data/sample.jsonl` if you also copied it locally per the rule below. Fill `MODEL_ENDPOINT` / `MODEL_DEPLOYMENT` (check G), and run-settings assignments. **Do not** pass `${data.col}` arguments. |
|
||||
| `data/sample.jsonl` | **Only** copy from the source `Input(path=...)` when (a) it points at a local file the agent can read, **and** (b) the user did not explicitly ask to keep the original input. For remote / `azureml://` / `Input(...)` already pointing at a workspace data asset, leave the source `Input(...)` unchanged in `submit_pipeline.py` and skip this file (do not invent a sample). Print a one-line note in the verdict table either way. |
|
||||
| `workflow.py` | **Default (consolidated):** already present in the target folder — do nothing. **Sibling-folder mode only:** copy from the source MAF folder. |
|
||||
|
||||
### 5. Validate & hand off
|
||||
|
||||
If no input-side TODO remains **and** a local jsonl file is available
|
||||
(either copied as `data/sample.jsonl` or already pointed at by the source
|
||||
`Input(path=...)`), run the local dry-run from the target folder:
|
||||
|
||||
```bash
|
||||
cd <maf-workflow-folder> # or <maf-workflow-folder>-prs in sibling mode
|
||||
python -c "
|
||||
import pandas as pd
|
||||
from types import SimpleNamespace
|
||||
import sys; sys.path.insert(0, 'src')
|
||||
from entry import init, run, shutdown
|
||||
init()
|
||||
ctx = SimpleNamespace(minibatch_index=0, global_row_index_lower_bound=0)
|
||||
print(run(pd.read_json('<path-to-local-jsonl>', lines=True), ctx))
|
||||
shutdown()
|
||||
"
|
||||
```
|
||||
|
||||
If the source `Input(path=...)` is a remote URI (datastore / data asset)
|
||||
**and** no local sample exists, **skip the dry-run** and tell the user
|
||||
the project will be exercised on the first AML submission instead.
|
||||
|
||||
If TODOs remain, skip the dry-run and tell the user which file to edit
|
||||
first. If the dry-run fails, consult
|
||||
[references/gotchas.md](references/gotchas.md), fix, retry.
|
||||
|
||||
Hand off `python submit_pipeline.py` with: the verdict table from step 3,
|
||||
the exact command to run, and a one-line description of what to look for
|
||||
in the streamed log (one JSONL row per input row in
|
||||
`outputs.flow_outputs/parallel_run_step.jsonl`).
|
||||
|
||||
---
|
||||
|
||||
## Related Skills
|
||||
|
||||
- [promptflow-to-maf](../promptflow-to-maf/SKILL.md) — convert the flow itself
|
||||
(run **before** this skill if not already done).
|
||||
- [maf-online-endpoint](../maf-online-endpoint/SKILL.md) — online (real-time)
|
||||
deployment of a MAF workflow. Use for request/response semantics rather
|
||||
than batch.
|
||||
- [maf-tracing](../maf-tracing/SKILL.md) — enable Application Insights tracing
|
||||
(`maf_prs/executor.py::_setup_tracing` already wires it up when
|
||||
`APPLICATIONINSIGHTS_CONNECTION_STRING` is set).
|
||||
|
||||
## References
|
||||
|
||||
- [pf-vs-maf-prs.md](references/pf-vs-maf-prs.md) — side-by-side mapping
|
||||
(PF auto-component → MAF hand-built) + PRS run-settings table.
|
||||
- [auto-derive-checks.md](references/auto-derive-checks.md) — the 10
|
||||
Phase 1.5 checks (A–J) + TODO stub template + verdict table format.
|
||||
- [gotchas.md](references/gotchas.md) — async loop reuse, mini-batch retry
|
||||
semantics, MSI / connection mapping, common dry-run failures.
|
||||
@@ -0,0 +1,101 @@
|
||||
# Azure ML Parallel Component that wraps a Microsoft Agent Framework workflow
|
||||
# as the per-row task. This file is the MAF equivalent of the component that
|
||||
# `load_component(flow.dag.yaml)` auto-generated for Prompt Flow PRS jobs.
|
||||
#
|
||||
# Settings here mirror the Prompt Flow PRS settings table 1:1. See the
|
||||
# accompanying SKILL.md for the mapping.
|
||||
$schema: https://azuremlschemas.azureedge.net/latest/parallelComponent.schema.json
|
||||
type: parallel
|
||||
name: maf_workflow_prs
|
||||
display_name: MAF Workflow as PRS
|
||||
version: 1
|
||||
description: >-
|
||||
Run a Microsoft Agent Framework workflow over a tabular data input as a
|
||||
parallel run step. Each row becomes one workflow invocation.
|
||||
|
||||
inputs:
|
||||
# `uri_file` is accepted by PRS when the PF compatibility flag set below
|
||||
# is present in `program_arguments`. This lets callers pass a plain
|
||||
# `.jsonl` file (including a `uri_file` produced by an upstream pipeline
|
||||
# step) directly. See gotchas.md #12.
|
||||
data:
|
||||
type: uri_file
|
||||
description: Single jsonl file; PRS parses each line into a row dict.
|
||||
# Each entry below replaces one PF "connection" field. Add or remove as
|
||||
# required by your workflow's chat client.
|
||||
model_endpoint:
|
||||
type: string
|
||||
description: Azure OpenAI / Foundry endpoint URL (e.g. https://my-foundry.services.ai.azure.com).
|
||||
model_deployment:
|
||||
type: string
|
||||
description: Deployment name (e.g. gpt-4o, gpt-35-turbo).
|
||||
api_version:
|
||||
type: string
|
||||
default: "2024-08-01-preview"
|
||||
optional: true
|
||||
|
||||
outputs:
|
||||
# Equivalent of PF's auto-generated `flow_outputs` port — PRS appends one
|
||||
# JSONL row per input row to this file.
|
||||
flow_outputs:
|
||||
type: uri_file
|
||||
# Equivalent of PF's auto-generated `debug_info` port — free-form folder for
|
||||
# the workflow to write intermediate artefacts to.
|
||||
debug_info:
|
||||
type: uri_folder
|
||||
|
||||
# Tell PRS which input is the dataset to be split into mini-batches.
|
||||
input_data: ${{inputs.data}}
|
||||
|
||||
# === PRS run settings (carry over from PF flow_node.* settings) =============
|
||||
mini_batch_size: "10"
|
||||
mini_batch_error_threshold: -1
|
||||
error_threshold: -1
|
||||
logging_level: DEBUG
|
||||
resources:
|
||||
instance_count: 1
|
||||
max_concurrency_per_instance: 2
|
||||
retry_settings:
|
||||
max_retries: 1
|
||||
timeout: 1200
|
||||
# ============================================================================
|
||||
|
||||
task:
|
||||
type: run_function
|
||||
# `code: ./` ships the whole project root so `workflow.py` (sibling of
|
||||
# `src/`) is uploaded together with `src/entry.py`. See gotchas.md §3
|
||||
# for context. Works for both layouts:
|
||||
# * consolidated : MAF folder containing workflow.py + src/ + ...
|
||||
# * sibling `-prs/`: copy of workflow.py at the same level as src/
|
||||
code: ./
|
||||
entry_script: src/entry.py
|
||||
environment:
|
||||
name: maf-prs-env
|
||||
version: 1
|
||||
image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest
|
||||
conda_file: env/conda.yml
|
||||
program_arguments: >-
|
||||
--amlbi_pf_enabled True
|
||||
--amlbi_pf_run_mode component
|
||||
--amlbi_file_format jsonl
|
||||
--amlbi_mini_batch_rows 1
|
||||
--model_endpoint ${{inputs.model_endpoint}}
|
||||
--model_deployment ${{inputs.model_deployment}}
|
||||
$[[--api_version ${{inputs.api_version}}]]
|
||||
--output_dir ${{outputs.debug_info}}
|
||||
# === PF compatibility flag set ============================================
|
||||
# `--amlbi_pf_enabled True` flips PRS's ArgValidator gate so that
|
||||
# `type: uri_file` is accepted.
|
||||
# `--amlbi_pf_run_mode component` PF "I am a flow component" signal; PRS
|
||||
# uses it to extract the `output` field
|
||||
# from each returned JSON string into the
|
||||
# appended jsonl line.
|
||||
# `--amlbi_file_format jsonl` tells PRS to parse the input file as
|
||||
# jsonl and dispatch row dicts (otherwise
|
||||
# uri_file → list[file_path] mini-batch).
|
||||
# `--amlbi_mini_batch_rows 1` switch from file-count to row-count
|
||||
# batching (paired with --amlbi_file_format).
|
||||
# ==========================================================================
|
||||
# Equivalent of PF's `parallel_run_step.jsonl` — PRS appends each entry of
|
||||
# the list returned by run() as a JSONL line to this output.
|
||||
append_row_to: ${{outputs.flow_outputs}}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Conda environment for the MAF PRS parallel component.
|
||||
#
|
||||
# The PF runtime image was inherited automatically when using
|
||||
# `load_component(flow.dag.yaml)`. For MAF we declare the env explicitly.
|
||||
# Add any packages your workflow imports below the marked section.
|
||||
#
|
||||
# All packages have lower-bound version pins because pip's resolver hits
|
||||
# `resolution-too-deep` on the AML image build host without them; without
|
||||
# constraints the image build aborts before the workflow ever runs.
|
||||
# See gotchas.md #14.
|
||||
name: maf-prs-env
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.11
|
||||
- pip
|
||||
- pip:
|
||||
# --- AML PRS runtime requirements ---
|
||||
- azureml-core>=1.59.0
|
||||
- azureml-mlflow>=1.59.0
|
||||
- azureml-dataset-runtime>=1.59.0
|
||||
- mltable>=1.6.0
|
||||
- setuptools<80
|
||||
- pandas>=2.2.0
|
||||
# --- Microsoft Agent Framework (GA) ---
|
||||
- agent-framework>=1.0.1
|
||||
- agent-framework-openai>=1.0.1
|
||||
- azure-identity>=1.19.0
|
||||
# --- Optional: Application Insights tracing (see maf-tracing skill) ---
|
||||
- azure-monitor-opentelemetry>=1.6.4
|
||||
# --- CUSTOMISE: your workflow's extra dependencies -------------------
|
||||
# Pin every addition with a lower bound (e.g. `package>=X.Y.Z`) to
|
||||
# keep pip's resolver below its depth limit on the AML build host.
|
||||
# - azure-ai-projects>=1.0.0
|
||||
# - azure-search-documents>=11.5.0
|
||||
# - <your-internal-package>>=1.0.0
|
||||
# ---------------------------------------------------------------------
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Azure ML Parallel Run Step (PRS) entry script for a Microsoft Agent Framework
|
||||
(MAF) workflow.
|
||||
|
||||
This file is the MAF equivalent of what `load_component(flow.dag.yaml)`
|
||||
auto-generated for Prompt Flow PRS jobs. PRS calls three top-level functions
|
||||
on this module per worker process:
|
||||
|
||||
init() -- once at boot
|
||||
run(mini_batch, context) -- once per mini-batch; returns list[str]
|
||||
shutdown() -- once before the worker exits
|
||||
|
||||
Keep this file thin. All real logic lives in `maf_prs/` so that adding new
|
||||
modes (e.g. bulk run, aggregation) does not require touching the PRS contract.
|
||||
|
||||
To customise row -> workflow input mapping, edit
|
||||
`maf_prs/executor.py::MafWorkflowExecutor.build_workflow_input`.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# `code: ./` in component.yaml uploads the project root, so PRS's sys.path
|
||||
# only contains the project root — `maf_prs` then resolves as
|
||||
# `src.maf_prs`, not `maf_prs`, and the import below would fail with
|
||||
# "No module named 'maf_prs'". Prepending this directory (`src/`) makes
|
||||
# the imports work regardless of how PRS happens to load the entry module
|
||||
# (`entry`, `src.entry`, …). See gotchas.md #13.
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
if str(_HERE) not in sys.path:
|
||||
sys.path.insert(0, str(_HERE))
|
||||
|
||||
from maf_prs.processor import create_processor # noqa: E402
|
||||
|
||||
_processor = None
|
||||
|
||||
|
||||
def init():
|
||||
"""Called once per worker process at boot."""
|
||||
global _processor
|
||||
# `Path(__file__).resolve().parents[1]` is the project root that contains
|
||||
# `workflow.py` (with `create_workflow()`). component.yaml uses
|
||||
# `code: ./` and `entry_script: src/entry.py`, so this layout puts the
|
||||
# workflow on sys.path both locally and on AML.
|
||||
_processor = create_processor(Path(__file__).resolve().parents[1])
|
||||
_processor.init()
|
||||
|
||||
|
||||
def run(mini_batch, context):
|
||||
"""Called once per mini-batch. Returns one JSON string per input row."""
|
||||
return _processor.process(mini_batch, context)
|
||||
|
||||
|
||||
def shutdown():
|
||||
"""Called once before the worker exits. Flushes tracing and closes the loop."""
|
||||
if _processor is not None:
|
||||
_processor.finalize()
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Per-workflow customisation hooks for the MAF PRS template.
|
||||
|
||||
This is the **only file most users need to edit**. `executor.py` and the
|
||||
rest of the `maf_prs/` package are generic plumbing and can be vendored
|
||||
unchanged across all converted workflows.
|
||||
|
||||
Three hooks are exposed:
|
||||
|
||||
setup(config) -- one-time worker setup; translate component
|
||||
inputs (config.model_endpoint, etc.) into
|
||||
whatever environment the workflow expects
|
||||
(env vars, file paths, ...). Default: no-op.
|
||||
|
||||
build_workflow_input(row) -- map one input row (dict) to the object the
|
||||
workflow's start executor expects.
|
||||
|
||||
serialize_output(output) -- convert the workflow's terminal output to a
|
||||
JSON-serialisable value that PRS will append
|
||||
to outputs.flow_outputs.
|
||||
|
||||
The skill agent fills `build_workflow_input` from the source PF mapping +
|
||||
the workflow's start handler signature when both are unambiguous (see
|
||||
auto-derive-checks.md A-D). Otherwise it leaves a `# TODO` stub.
|
||||
|
||||
Imports of `workflow` work at module load because `executor.init()` adds
|
||||
the project root to `sys.path` *before* importing this module.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# CUSTOMISE: import the typed input class(es) your workflow's start executor
|
||||
# expects. Leave commented out if the start handler accepts a free-form dict.
|
||||
# from workflow import FlowInput
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CUSTOMISE #0: one-time worker setup
|
||||
# ---------------------------------------------------------------------------
|
||||
# Called once per worker process (after sys.path is wired up, before any
|
||||
# row is processed). Use this to translate component inputs into whatever
|
||||
# the workflow needs at construction time.
|
||||
#
|
||||
# Examples:
|
||||
# if config.model_endpoint:
|
||||
# os.environ["AZURE_OPENAI_ENDPOINT"] = config.model_endpoint
|
||||
# if config.model_deployment:
|
||||
# os.environ["AZURE_OPENAI_DEPLOYMENT"] = config.model_deployment
|
||||
# ---------------------------------------------------------------------------
|
||||
def setup(config) -> None:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CUSTOMISE #1: row -> workflow input
|
||||
# ---------------------------------------------------------------------------
|
||||
# `row` is the full mini-batch row as a dict. Return whatever the workflow's
|
||||
# first executor's @handler expects.
|
||||
#
|
||||
# Examples:
|
||||
# PF: flow_node(url="${data.url}")
|
||||
# MAF: return row["url"]
|
||||
#
|
||||
# PF: flow_node(question="${data.q}", history="${data.history}")
|
||||
# MAF: return {"question": row["q"], "history": row["history"]}
|
||||
#
|
||||
# PF: (n/a -- workflow takes a typed object)
|
||||
# MAF: return FlowInput(url=row["url"])
|
||||
# ---------------------------------------------------------------------------
|
||||
def build_workflow_input(row: dict) -> Any:
|
||||
# Default: pass the entire row through. Safe when the workflow's
|
||||
# first executor accepts a free-form dict; otherwise replace this.
|
||||
return row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CUSTOMISE #2: workflow output -> JSON-serialisable value
|
||||
# ---------------------------------------------------------------------------
|
||||
# `output` is the last item of `WorkflowRunResult.get_outputs()`. The
|
||||
# default below duck-types the most common shapes (dicts, lists, primitives,
|
||||
# objects with `.text`) and falls back to `str(output)`. Override only if
|
||||
# your workflow emits a custom payload that the duck-typed fallback mangles.
|
||||
# ---------------------------------------------------------------------------
|
||||
def serialize_output(output: Any) -> Any:
|
||||
if output is None:
|
||||
return None
|
||||
if isinstance(output, (dict, list, str, int, float, bool)):
|
||||
return output
|
||||
if hasattr(output, "text"):
|
||||
return output.text
|
||||
return str(output)
|
||||
@@ -0,0 +1 @@
|
||||
"""MAF PRS plumbing — mirrors promptflow-parallel's processor/executor split."""
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
PRS argument parsing. Equivalent of promptflow-parallel's `_config/parser.py`,
|
||||
minus the `--pf_input_*` column-mapping flags (column mapping is now done
|
||||
inside `executor.build_workflow_input`).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class MafPrsConfig:
|
||||
"""Resolved PRS config for one worker. Add fields as your workflow needs."""
|
||||
model_endpoint: Optional[str] = None
|
||||
model_deployment: Optional[str] = None
|
||||
api_version: str = "2024-08-01-preview"
|
||||
debug_output_dir: Optional[Path] = None
|
||||
|
||||
|
||||
def parse_args(argv: Optional[List[str]] = None) -> MafPrsConfig:
|
||||
"""Parse PRS `program_arguments` from component.yaml.
|
||||
|
||||
Uses `parse_known_args` so PRS-injected arguments (logging, telemetry,
|
||||
etc.) don't crash us — same pattern as promptflow-parallel.
|
||||
"""
|
||||
parser = ArgumentParser()
|
||||
# One --flag per component input that the workflow needs. These typically
|
||||
# come from `flow_node(connections=...)` in the original PF PRS submission
|
||||
# and are surfaced as component inputs in component.yaml.
|
||||
parser.add_argument("--model_endpoint")
|
||||
parser.add_argument("--model_deployment")
|
||||
parser.add_argument("--api_version", default="2024-08-01-preview")
|
||||
# Bound to outputs.debug_info — the workflow may write intermediate
|
||||
# artefacts here. Equivalent of PF's debug_info port.
|
||||
parser.add_argument("--output_dir", type=Path, dest="debug_output_dir")
|
||||
parsed, _unknown = parser.parse_known_args(argv)
|
||||
return MafPrsConfig(**vars(parsed))
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Per-row execution of a MAF workflow. **Generic** — should not need editing.
|
||||
|
||||
Equivalent of promptflow-parallel's `ComponentRunExecutor`:
|
||||
PF MAF (this file)
|
||||
------------------------------- -------------------------------
|
||||
FlowExecutor.create(flow_dag, conns) from workflow import create_workflow
|
||||
flow_executor.exec_line(inputs, index) await workflow.run(hooks.build_workflow_input(row))
|
||||
FlowExecutor.apply_inputs_mapping hooks.build_workflow_input(row) ← in hooks.py
|
||||
persist_multimedia_data hooks.serialize_output(output) ← in hooks.py
|
||||
|
||||
All per-workflow customisation lives in `src/hooks.py`. This file only
|
||||
provides the generic per-row driver.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .config import MafPrsConfig
|
||||
|
||||
logger = logging.getLogger("maf-prs.executor")
|
||||
|
||||
|
||||
class MafWorkflowExecutor:
|
||||
"""Drives a MAF workflow once per input row."""
|
||||
|
||||
def __init__(self, working_dir: Path, config: MafPrsConfig):
|
||||
self._working_dir = working_dir
|
||||
self._config = config
|
||||
# Factory only — never cache an instance. MAF workflows do not
|
||||
# support concurrent run() on the same instance.
|
||||
self._create_workflow = None
|
||||
# User-provided customisation hooks (src/hooks.py).
|
||||
self._hooks = None
|
||||
|
||||
# ---- init ---------------------------------------------------------
|
||||
def init(self) -> None:
|
||||
if str(self._working_dir) not in sys.path:
|
||||
sys.path.insert(0, str(self._working_dir))
|
||||
|
||||
# `hooks` is imported *after* sys.path is wired so it can do
|
||||
# `from workflow import ...` at module load.
|
||||
import hooks # noqa: E402
|
||||
|
||||
hooks.setup(self._config)
|
||||
self._hooks = hooks
|
||||
|
||||
# The MAF workflow project must export a `create_workflow()` factory
|
||||
# (per the promptflow-to-maf skill rule). If your file lives
|
||||
# somewhere else, change this import.
|
||||
from workflow import create_workflow # noqa: E402
|
||||
self._create_workflow = create_workflow
|
||||
|
||||
self._setup_tracing()
|
||||
|
||||
# ---- per-row ------------------------------------------------------
|
||||
async def execute(self, row: dict, row_number: int) -> dict:
|
||||
"""Run the workflow for one row. Catches exceptions per-row so a
|
||||
single bad row does not poison the whole mini-batch (PF behaviour was
|
||||
to fail-and-retry the batch; tune via `error_threshold` /
|
||||
`mini_batch_error_threshold` in component.yaml if you want that)."""
|
||||
assert self._create_workflow is not None, "init() was not called"
|
||||
workflow = self._create_workflow()
|
||||
try:
|
||||
wf_input = self._hooks.build_workflow_input(row)
|
||||
result = await workflow.run(wf_input)
|
||||
outputs = result.get_outputs() or [None]
|
||||
return {
|
||||
"line_number": row_number,
|
||||
"input": row,
|
||||
"output": self._hooks.serialize_output(outputs[0]),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001 — per-row isolation
|
||||
logger.exception("Row %d failed", row_number)
|
||||
return {
|
||||
"line_number": row_number,
|
||||
"input": row,
|
||||
"output": None,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
|
||||
# ---- finalize -----------------------------------------------------
|
||||
def finalize(self) -> None:
|
||||
"""Close any cached chat clients / flush OTel here."""
|
||||
# No-op by default; the workflow factory builds clients per-row, so
|
||||
# there is nothing to close at the executor level. If your hooks.py
|
||||
# caches a shared client, add a `teardown()` hook and call it here.
|
||||
return None
|
||||
|
||||
# ---- internal -----------------------------------------------------
|
||||
def _setup_tracing(self) -> None:
|
||||
"""Optional Application Insights tracing. See the maf-tracing skill."""
|
||||
conn = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
|
||||
if not conn:
|
||||
return
|
||||
try:
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
configure_azure_monitor(connection_string=conn)
|
||||
configure_otel_providers()
|
||||
logger.info("Application Insights tracing enabled.")
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"APPLICATIONINSIGHTS_CONNECTION_STRING is set but "
|
||||
"azure-monitor-opentelemetry is not installed; skipping."
|
||||
)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
Mini-batch orchestration.
|
||||
|
||||
Equivalent of promptflow-parallel's `AbstractParallelRunProcessor`:
|
||||
PF MAF (this file)
|
||||
----------------------------------------- --------------------------------
|
||||
create_processor(working_dir, args) create_processor(working_dir, args)
|
||||
Row.from_dict(data, row_number=base+i) executor.execute(row, base+i)
|
||||
self._executor.execute(row) await executor.execute(...)
|
||||
json.dumps(result_dict, cls=DataClassEncoder) json.dumps(result, default=str)
|
||||
AggregationFinalizer + _ComponentRunFinalizer executor.finalize() + loop.close()
|
||||
|
||||
The processor:
|
||||
* holds the asyncio event loop (created once in init(), reused across all
|
||||
run() calls — see references/gotchas.md #2 for why);
|
||||
* dispatches rows to the executor with `asyncio.gather` for in-process row
|
||||
concurrency (bounded by `max_concurrency_per_instance` in component.yaml);
|
||||
* preserves PRS row-order and row-count so the appended JSONL lines match
|
||||
the input data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from .config import MafPrsConfig, parse_args
|
||||
from .executor import MafWorkflowExecutor
|
||||
|
||||
logger = logging.getLogger("maf-prs.processor")
|
||||
|
||||
|
||||
class MafWorkflowProcessor:
|
||||
"""PRS processor — drives one MafWorkflowExecutor per worker process."""
|
||||
|
||||
def __init__(self, working_dir: Path, args: Optional[List[str]] = None):
|
||||
self._working_dir = working_dir
|
||||
self._args = args
|
||||
self._cfg: Optional[MafPrsConfig] = None
|
||||
self._executor: Optional[MafWorkflowExecutor] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
# ---- PRS: init ----------------------------------------------------
|
||||
def init(self) -> None:
|
||||
self._cfg = parse_args(self._args)
|
||||
# Re-use a single event loop across all run() invocations. Calling
|
||||
# asyncio.run() per row leaks transports inside many Azure SDK clients.
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
self._executor = MafWorkflowExecutor(self._working_dir, self._cfg)
|
||||
self._executor.init()
|
||||
if self._cfg.debug_output_dir:
|
||||
self._cfg.debug_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("MAF PRS processor initialised: cfg=%s", self._cfg)
|
||||
|
||||
# ---- PRS: process(mini_batch, context) ----------------------------
|
||||
def process(self, mini_batch: Any, context: Any) -> List[str]:
|
||||
"""Returns one JSON string per input row. PRS appends each as a
|
||||
line to outputs.flow_outputs (the AML equivalent of PF's
|
||||
`parallel_run_step.jsonl`)."""
|
||||
assert self._executor is not None and self._loop is not None, "init() was not called"
|
||||
base = getattr(context, "global_row_index_lower_bound", 0)
|
||||
minibatch_id = getattr(context, "minibatch_index", "?")
|
||||
rows = list(self._iter_rows(mini_batch))
|
||||
logger.info("mini_batch %s: %d rows (base row_number=%d)", minibatch_id, len(rows), base)
|
||||
|
||||
async def _run_all() -> List[dict]:
|
||||
return await asyncio.gather(
|
||||
*(self._executor.execute(row, base + i) for i, row in enumerate(rows))
|
||||
)
|
||||
|
||||
results = self._loop.run_until_complete(_run_all())
|
||||
return [json.dumps(r, default=str) for r in results]
|
||||
|
||||
# ---- PRS: shutdown / finalize ------------------------------------
|
||||
def finalize(self) -> None:
|
||||
try:
|
||||
if self._executor is not None:
|
||||
self._executor.finalize()
|
||||
finally:
|
||||
if self._loop is not None and not self._loop.is_closed():
|
||||
self._loop.run_until_complete(self._loop.shutdown_asyncgens())
|
||||
self._loop.close()
|
||||
|
||||
# ---- helpers ------------------------------------------------------
|
||||
@staticmethod
|
||||
def _iter_rows(mini_batch: Any) -> Iterable[dict]:
|
||||
"""Normalise the PRS mini_batch into a stream of row dicts.
|
||||
|
||||
PRS dispatches one of three shapes:
|
||||
* `pandas.DataFrame` for tabular inputs.
|
||||
* `list[dict]` when input is `uri_file` + `--amlbi_file_format jsonl`
|
||||
(PRS parses the jsonl and hands each row as a dict). See
|
||||
gotchas.md #12 for the PF-compat workaround that uses this path.
|
||||
* `list[str]` of file paths when input is `uri_folder` of opaque files.
|
||||
"""
|
||||
if isinstance(mini_batch, pd.DataFrame):
|
||||
yield from mini_batch.to_dict(orient="records")
|
||||
return
|
||||
for item in mini_batch:
|
||||
if isinstance(item, dict):
|
||||
yield item
|
||||
else:
|
||||
yield {"path": str(item)}
|
||||
|
||||
|
||||
def create_processor(working_dir: Path, args: Optional[List[str]] = None) -> MafWorkflowProcessor:
|
||||
return MafWorkflowProcessor(working_dir, args)
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Submit a MAF-workflow PRS pipeline to Azure ML.
|
||||
|
||||
This file is the MAF-side analogue of the Prompt Flow PRS submission shown in
|
||||
`examples/tutorials/run-flow-with-pipeline/pipeline.ipynb` section 3.2.1.
|
||||
|
||||
Mapping vs. the original PF script:
|
||||
|
||||
PF MAF (this file)
|
||||
---------------------------------- -----------------------------------
|
||||
load_component("flow.dag.yaml") --> load_component("component.yaml")
|
||||
flow_node(url="${data.url}", ...) --> workflow_component(data=..., model_endpoint=...,
|
||||
model_deployment=...)
|
||||
flow_node(connections={...}) --> workflow_component(model_endpoint=..., ...)
|
||||
+ Managed Identity / Key Vault for secrets
|
||||
flow_node.compute / .resources / --> identical assignments on the
|
||||
.mini_batch_size / .retry_settings component instance returned by
|
||||
/ .logging_level / etc. the @pipeline DSL
|
||||
|
||||
Run:
|
||||
python submit_pipeline.py
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from azure.ai.ml import Input, MLClient, Output, load_component
|
||||
from azure.ai.ml.constants import AssetTypes
|
||||
from azure.ai.ml.dsl import pipeline
|
||||
from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Workspace handle (preserve whatever the original PF script used).
|
||||
# ---------------------------------------------------------------------------
|
||||
try:
|
||||
credential = DefaultAzureCredential()
|
||||
credential.get_token("https://management.azure.com/.default")
|
||||
except Exception: # noqa: BLE001
|
||||
credential = InteractiveBrowserCredential()
|
||||
|
||||
ml_client = MLClient.from_config(credential=credential)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Load the parallel component (replaces `load_component(flow.dag.yaml)`).
|
||||
# ---------------------------------------------------------------------------
|
||||
HERE = Path(__file__).parent
|
||||
workflow_component = load_component(str(HERE / "component.yaml"))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Pipeline-level inputs / outputs (carry over from the original PF script).
|
||||
# ---------------------------------------------------------------------------
|
||||
data_input = Input(
|
||||
# PRESERVE the source `Input(path=..., type=..., mode=...)` from the
|
||||
# original PF submission VERBATIM — same path (local file, datastore URI,
|
||||
# or registered data asset), same type (URI_FILE works thanks to the PF
|
||||
# compat flag set in component.yaml), same mode. The skill agent only
|
||||
# rewrites this block when the user explicitly asks for a self-contained
|
||||
# local sample under `data/`.
|
||||
path="<COPIED FROM SOURCE Input(path=...)>",
|
||||
type=AssetTypes.URI_FILE,
|
||||
mode="mount",
|
||||
)
|
||||
|
||||
pipeline_output = Output(
|
||||
# path="azureml://datastores/<data_store_name>/paths/<path>",
|
||||
type=AssetTypes.URI_FOLDER,
|
||||
mode="rw_mount",
|
||||
)
|
||||
|
||||
# CUSTOMISE: values that originally came from `flow_node(connections={...})`.
|
||||
MODEL_ENDPOINT = "https://<your-foundry-resource>.services.ai.azure.com"
|
||||
MODEL_DEPLOYMENT = "gpt-4o"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Pipeline definition (mirrors the @pipeline() function in the PF script).
|
||||
# ---------------------------------------------------------------------------
|
||||
@pipeline()
|
||||
def pipeline_func_with_workflow(
|
||||
pipeline_input_data: Input,
|
||||
parallel_node_count: int = 1,
|
||||
):
|
||||
workflow_node = workflow_component(
|
||||
data=pipeline_input_data,
|
||||
model_endpoint=MODEL_ENDPOINT,
|
||||
model_deployment=MODEL_DEPLOYMENT,
|
||||
)
|
||||
|
||||
# === Carry over PF run settings 1:1 =====================================
|
||||
workflow_node.environment_variables = {
|
||||
# If your data is not jsonl, set the right format expected by entry.py
|
||||
# when reading mini-batches (entry.py uses pandas, so jsonl/csv/tsv all
|
||||
# work; this env var is informational for downstream tools).
|
||||
"PF_INPUT_FORMAT": "jsonl",
|
||||
}
|
||||
workflow_node.compute = "cpu-cluster"
|
||||
workflow_node.resources = {"instance_count": parallel_node_count}
|
||||
workflow_node.mini_batch_size = 5
|
||||
workflow_node.max_concurrency_per_instance = 2
|
||||
workflow_node.retry_settings = {"max_retries": 1, "timeout": 1200}
|
||||
workflow_node.error_threshold = -1
|
||||
workflow_node.mini_batch_error_threshold = -1
|
||||
workflow_node.logging_level = "DEBUG"
|
||||
|
||||
# When instance_count > 1, both PRS outputs must use mount mode (same rule
|
||||
# as the PF flow component).
|
||||
workflow_node.outputs.flow_outputs.mode = "mount"
|
||||
workflow_node.outputs.debug_info.mode = "mount"
|
||||
# ========================================================================
|
||||
|
||||
return {"flow_result_folder": workflow_node.outputs.flow_outputs}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Submit.
|
||||
# ---------------------------------------------------------------------------
|
||||
pipeline_job_def = pipeline_func_with_workflow(pipeline_input_data=data_input)
|
||||
pipeline_job_def.outputs.flow_result_folder = pipeline_output
|
||||
|
||||
if __name__ == "__main__":
|
||||
pipeline_job_run = ml_client.jobs.create_or_update(
|
||||
pipeline_job_def,
|
||||
experiment_name="maf_workflow_prs_job",
|
||||
)
|
||||
print(f"Submitted job: {pipeline_job_run.name}")
|
||||
print(f"Studio URL: {pipeline_job_run.studio_url}")
|
||||
ml_client.jobs.stream(pipeline_job_run.name)
|
||||
@@ -0,0 +1,87 @@
|
||||
# Phase 1.5 — Auto-derive Verdict Checks
|
||||
|
||||
Run these checks **before** generating files to decide which fields the
|
||||
agent can fill automatically vs. which must be left as `# TODO` stubs for
|
||||
the user. Print the resulting verdict table to the user before writing
|
||||
any code.
|
||||
|
||||
> Rule of thumb: **never invent values.** When in doubt, emit a TODO with
|
||||
> a comment quoting the original PF source and the reason auto-derivation
|
||||
> stopped.
|
||||
|
||||
---
|
||||
|
||||
## Input side — `hooks.build_workflow_input(row)`
|
||||
|
||||
The first failure stops auto-derivation; the function becomes a TODO stub.
|
||||
|
||||
| # | Check | What "enough info" looks like | If missing |
|
||||
|---|---|---|---|
|
||||
| **A** | Mapping is parseable | Every kwarg in `flow_node(...)` that is **not** a known PRS setting (`compute`, `mini_batch_size`, `connections`, etc.) has a value that fully matches `r"\$\{data\.([\w\.]+)\}"`. | TODO: "PF mapping uses non-trivial expression `<value>` — fill manually." |
|
||||
| **B** | Mapping is non-empty | At least one `${data.col}` was found. | Default to `return row` (pass-through). No TODO needed if the workflow's first executor accepts a free-form `dict`. |
|
||||
| **C** | Start handler is typed | `inspect.signature(start_executor.handler)` shows a typed first non-`self`, non-`ctx` parameter. Accepted: `str`, `int`, `float`, `bool`, `dict`, `dict[str, Any]`, a `@dataclass`, a `pydantic.BaseModel` subclass, or `agent_framework.ChatMessage`. | TODO: "Workflow start handler input is `Any` / untyped — cannot infer shape; map row to handler input manually." |
|
||||
| **D** | Mapping fields fit handler | The PF target field names match the handler input's fields (for `dict` / dataclass / Pydantic), or there is exactly one PF mapping (for scalar handler input). | TODO: "PF mapping fields `<a, b>` do not match handler `<C>` fields `<x, y>` — fill manually." |
|
||||
|
||||
When A–D **all** pass, emit one of these bodies:
|
||||
|
||||
| Handler input type | Mapping shape | Generated body |
|
||||
|---|---|---|
|
||||
| `str` / `int` / `float` / `bool` | exactly one `${data.col}` | `return row["col"]` |
|
||||
| `dict` / `dict[str, Any]` | N `${data.col_i}` → N target fields | `return {"f1": row["c1"], ...}` |
|
||||
| `@dataclass` / `BaseModel` / `ChatMessage` | N `${data.col_i}` → N target fields matching the model | `return MyInput(f1=row["c1"], ...)` (plus an `import` line at the top of `hooks.py`) |
|
||||
|
||||
### TODO stub template
|
||||
|
||||
```python
|
||||
def build_workflow_input(row: dict):
|
||||
# TODO: PF mapping was flow_node(input="${data.url}")
|
||||
# but the workflow's start executor handler is untyped (Any),
|
||||
# so the input shape cannot be inferred. Replace this with a
|
||||
# call that returns the object your start executor expects.
|
||||
raise NotImplementedError("Fill build_workflow_input for your workflow.")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Output side — `hooks.serialize_output(output)`
|
||||
|
||||
| # | Check | What "enough info" looks like | If missing |
|
||||
|---|---|---|---|
|
||||
| **E** | Terminal output type is inspectable | The workflow's terminal executor calls `ctx.set_output(<typed value>)` or yields `WorkflowOutputEvent(payload=<typed value>)` where the payload type is a class the agent can resolve via AST. | Keep the **default** `serialize_output` from the template (the duck-typed fallback works for most cases). Add a comment: `# auto: kept default — verify against your workflow output shape.` Do **not** leave a hard TODO. |
|
||||
|
||||
---
|
||||
|
||||
## Connection inputs — `component.yaml` + `submit_pipeline.py`
|
||||
|
||||
| # | Check | What "enough info" looks like | If missing |
|
||||
|---|---|---|---|
|
||||
| **F** | Connection kwargs identifiable | `flow_node(connections={...})` (or per-node kwargs like `connection=`, `deployment_name=`, `model=`, `api_version=`) are present in the source script. | `# TODO` in `component.yaml::inputs` and a `# TODO` for `MODEL_ENDPOINT` / `MODEL_DEPLOYMENT` constants in `submit_pipeline.py`. |
|
||||
| **G** | Endpoint URL resolvable | The PF connection name maps to a known Azure resource the user has already deployed (or is provided in the audit). | Leave the `MODEL_ENDPOINT = "https://<your-foundry-resource>..."` placeholder and tell the user what to fill. **Never** invent an endpoint URL. |
|
||||
|
||||
---
|
||||
|
||||
## Data input/output ports
|
||||
|
||||
| # | Check | What "enough info" looks like | If missing |
|
||||
|---|---|---|---|
|
||||
| **H** | Input data path/type/mode | `Input(path=..., type=..., mode=...)` literally appears in the source script (or a notebook cell). | Leave the `data_input = Input(...)` block in `submit_pipeline.py` with a `# TODO` placeholder. |
|
||||
| **I** | Local sample copy needed? | The user wants the generated project to be self-contained for local dry-run **and** the source `Input(path=...)` resolves to a local file the agent can read. | **Default: do nothing.** Reuse the source `Input(...)` verbatim in `submit_pipeline.py` and **do not** create `data/sample.jsonl`. Add a note in the verdict table: "reused source data input; local dry-run requires you to point `data/` at a small local file." Only copy when both conditions hold; never invent a sample. |
|
||||
| **J** | Pipeline output path | `pipeline_output = Output(path=..., ...)` is set with a literal datastore URI. | Keep the template's commented-out `path=` line and tell the user it will land in the default workspace blobstore. |
|
||||
|
||||
---
|
||||
|
||||
## Verdict table (show to the user)
|
||||
|
||||
After running checks A–J, print a table like this **before** writing any
|
||||
files. The same table doubles as the change log handed to the user at
|
||||
the end.
|
||||
|
||||
| Method / file | Status | Notes |
|
||||
|---|---|---|
|
||||
| `hooks.build_workflow_input` | ✅ auto-filled | maps `${data.url}` → `row["url"]` |
|
||||
| `hooks.serialize_output` | ✅ default kept | output type `str` matches duck-typed fallback |
|
||||
| `hooks.setup` | ✅ auto-filled | translates `model_endpoint` / `model_deployment` to env vars |
|
||||
| `component.yaml::inputs` | ✅ auto-filled | from `connection="aoai_conn", deployment_name="gpt-4o"` |
|
||||
| `submit_pipeline.py::MODEL_ENDPOINT` | ⚠️ TODO | PF connection name `aoai_conn` not resolvable to a deployed resource |
|
||||
| `submit_pipeline.py::data_input` | ✅ auto-filled | preserved source `Input(path="azureml://...", type=URI_FILE, mode="ro_mount")` verbatim |
|
||||
| `data/sample.jsonl` | ➖ not created | reused source data input; provide a local file path here only if you want a self-contained dry-run |
|
||||
@@ -0,0 +1,337 @@
|
||||
# MAF PRS — Common Gotchas
|
||||
|
||||
> Read this when something fails during local dry-run (Phase 3) or when the
|
||||
> first AML submission fails (Phase 4). Ordered by frequency.
|
||||
|
||||
## 1. `RuntimeError: Workflow is already running`
|
||||
|
||||
**Cause:** A single MAF workflow instance was reused across rows or across
|
||||
`run()` calls.
|
||||
|
||||
**Fix:** Build the workflow inside `MafWorkflowExecutor.execute(row, ...)`
|
||||
(per-row) — never cache `self._workflow = create_workflow()` in
|
||||
`executor.init()`. The template `executor.py` only caches the **factory**
|
||||
(`self._create_workflow`), not an instance; if you removed that distinction
|
||||
for "performance", put it back.
|
||||
|
||||
## 2. `RuntimeError: There is no current event loop in thread 'Dummy-N'`
|
||||
|
||||
**Cause:** PRS spawns worker threads/processes; the loop created in `init()`
|
||||
on one thread is not visible to another.
|
||||
|
||||
**Fix:** The template `processor.init()` uses `asyncio.new_event_loop()` +
|
||||
`asyncio.set_event_loop(loop)`, and `processor.process()` calls
|
||||
`self._loop.run_until_complete(...)`. Do not switch to `asyncio.run(...)` —
|
||||
it creates and tears down a loop per call, which leaks transports inside many
|
||||
Azure SDK clients. `processor.finalize()` closes the loop on shutdown.
|
||||
|
||||
## 3. `ImportError: cannot import name 'create_workflow' from 'workflow'`
|
||||
|
||||
**Cause:** Either the workflow file does not export a factory, or
|
||||
`component.yaml` is not set up so the workflow ends up on `sys.path`.
|
||||
|
||||
**Fix:**
|
||||
- Confirm `workflow.py` exports `create_workflow()` (this is also a
|
||||
`promptflow-to-maf` rule — see rule 11 of that skill).
|
||||
- The template `executor.init()` does `sys.path.insert(0, working_dir)` and
|
||||
`entry.init()` passes `Path(__file__).resolve().parents[1]` as the working
|
||||
dir, on the assumption that the project layout is `<root>/src/entry.py`,
|
||||
`<root>/src/hooks.py`, `<root>/src/maf_prs/...`, and `<root>/workflow.py`.
|
||||
If you flatten the layout, update the `parents[N]` index in `entry.py`.
|
||||
- `component.yaml` must use `code: ./` and `entry_script: src/entry.py`
|
||||
(not `code: ./src`) so the project root — which contains `workflow.py`
|
||||
alongside `src/` — is uploaded to AML. AML packages whatever directory
|
||||
`code:` points to and uploads it as the task code; files **outside**
|
||||
that directory are not shipped, so `code: ./src` would leave
|
||||
`workflow.py` behind.
|
||||
- This works for both supported layouts:
|
||||
* **Consolidated** (default): the PRS files (`src/`, `component.yaml`,
|
||||
…) live inside the existing MAF workflow folder, next to the
|
||||
workflow's own `workflow.py`.
|
||||
* **Sibling** (`<maf-folder>-prs/`): `workflow.py` is **copied** into
|
||||
the new folder so `code: ./` ships it.
|
||||
|
||||
## 4. PRS retries the whole mini-batch on a single bad row
|
||||
|
||||
**Cause:** The default behaviour: if `run()` raises, PRS retries the entire
|
||||
mini-batch (matches PF semantics).
|
||||
|
||||
**Fix:**
|
||||
- The template `executor.execute(...)` catches per-row exceptions and records
|
||||
them in the result dict so a single bad row does not poison the batch. If
|
||||
you want PRS to retry on transient errors, **re-raise** specific exception
|
||||
types (e.g. throttling 429 from the LLM) inside `execute()`.
|
||||
- Tune `mini_batch_size` down to keep retry cost small.
|
||||
- Use `error_threshold` (per-row) and `mini_batch_error_threshold` to bound
|
||||
total failures before aborting the job. `-1` disables both.
|
||||
|
||||
## 5. Output JSONL has wrong number of rows
|
||||
|
||||
**Cause:** `run()` returned fewer or more entries than `len(mini_batch)`.
|
||||
|
||||
**Fix:** PRS expects one entry per input row. The template
|
||||
`processor.process()` returns `asyncio.gather(*per_row)` which preserves
|
||||
order and length. If you do any filtering inside `executor.execute`, return
|
||||
a placeholder dict (e.g. `{"line_number": n, "input": row, "output": None,
|
||||
"skipped": True}`) instead of dropping rows.
|
||||
## 6. `azure.identity.CredentialUnavailableError` on AML compute
|
||||
|
||||
**Cause:** The compute does not have a Managed Identity assigned, or the
|
||||
identity lacks the role on the LLM endpoint resource.
|
||||
|
||||
**Fix:**
|
||||
- Assign a Managed Identity to the AML compute cluster (system-assigned is
|
||||
simplest).
|
||||
- Grant it `Cognitive Services OpenAI User` (or equivalent Foundry role) on
|
||||
the AI Services resource scope.
|
||||
- Do **not** ship API keys via `program_arguments` — they end up in job
|
||||
metadata and logs.
|
||||
|
||||
## 7. `pip` install fails on `agent-framework` inside the PRS env
|
||||
|
||||
**Cause:** Wrong Python version or missing build deps.
|
||||
|
||||
**Fix:**
|
||||
- Pin `python=3.11` in `conda.yml` (default in the template). MAF supports
|
||||
3.10+ but 3.11 is the safest match for current `agent-framework` wheels.
|
||||
- Use the `mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest` base
|
||||
image (default in the template) — it has `gcc` available for any wheels
|
||||
that need to compile.
|
||||
|
||||
## 8. "No outputs produced" on every row
|
||||
|
||||
**Cause:** `result.get_outputs()` returned `[]` because the workflow only
|
||||
emits intermediate events, not a final `WorkflowOutputEvent`.
|
||||
|
||||
**Fix:** Confirm the workflow's last executor calls
|
||||
`ctx.set_output(...)` or yields a `WorkflowOutputEvent`. If the workflow
|
||||
streams text instead, switch `executor.execute` to use `workflow.run_stream(...)`
|
||||
and aggregate `AgentRunUpdateEvent` deltas before serialising.
|
||||
|
||||
## 9. Mini-batch shape mismatch (DataFrame vs. file list)
|
||||
|
||||
**Cause:** `entry.py` was tested against a jsonl file but the component is
|
||||
fed a `uri_folder` of opaque files (or vice versa).
|
||||
|
||||
**Fix:** The template `processor._iter_rows()` handles both shapes. If you
|
||||
customised it, make sure both branches are tested. As a quick check, log
|
||||
`type(mini_batch).__name__` at the top of `processor.process()`.
|
||||
|
||||
## 10. Job runs locally but hangs on AML
|
||||
|
||||
**Cause:** The workflow is making an outbound call that the AML compute
|
||||
network policy blocks (e.g. private-endpoint-only Foundry resource, missing
|
||||
DNS zone link, or NSG denial).
|
||||
|
||||
**Fix:** This is an Azure networking issue, not a MAF issue. Verify with
|
||||
`az network watcher test-connectivity` from a debug pod on the cluster, or
|
||||
attach Application Insights tracing (`executor._setup_tracing` already wires
|
||||
it up if `APPLICATIONINSIGHTS_CONNECTION_STRING` is set as a deployment env
|
||||
var) and look for connection timeouts.
|
||||
|
||||
## 11. `line_number` missing from output rows (breaks downstream PF eval)
|
||||
|
||||
**Cause:** `entry.run()` was defined as `def run(mini_batch):` (one
|
||||
parameter) instead of `def run(mini_batch, context):`. Without `context`
|
||||
the processor cannot read `global_row_index_lower_bound`, so `line_number`
|
||||
falls back to `0` for every mini-batch — batch-eval tools that join
|
||||
input rows to output rows by `line_number` will silently produce wrong
|
||||
results.
|
||||
|
||||
**Fix:** Keep the template `run(mini_batch, context)` signature. PRS
|
||||
always passes both arguments; the second was optional only in very old
|
||||
PRS runtime versions.
|
||||
|
||||
## 12. `ArgumentException: Input format UriFile is not supported.`
|
||||
|
||||
**Cause:** The pipeline's `data` Input was declared as
|
||||
`Input(type=AssetTypes.URI_FILE, ...)` and `program_arguments` does not
|
||||
carry the PF compatibility flag set, so PRS's
|
||||
`ArgValidator._assert_uri_file_enabled` rejects `uri_file` at boot.
|
||||
|
||||
**The PRS runtime, however, has an undocumented compatibility mode** that
|
||||
PF used to ship `uri_file` jsonl inputs. When the parallel component's
|
||||
`program_arguments` carries the **PF compatibility flag set**, PRS:
|
||||
|
||||
1. Skips the `uri_file` validator gate.
|
||||
2. Parses the jsonl file line-by-line and dispatches **`list[dict]`**
|
||||
row mini-batches into `entry.run(mini_batch, context)`.
|
||||
3. Reads each returned JSON string and writes only the `output` field
|
||||
into `parallel_run_step.jsonl` (matching PF's "your flow's output IS
|
||||
the row output" semantic).
|
||||
|
||||
**Fix (default for this skill):** add the four flags below to
|
||||
`component.yaml::task.program_arguments` and set `data.type: uri_file`:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
data:
|
||||
type: uri_file
|
||||
description: Single jsonl file; PRS parses each line into a row dict.
|
||||
|
||||
task:
|
||||
program_arguments: >-
|
||||
--amlbi_pf_enabled True
|
||||
--amlbi_pf_run_mode component
|
||||
--amlbi_file_format jsonl
|
||||
--amlbi_mini_batch_rows 1
|
||||
--output_dir ${{outputs.debug_info}}
|
||||
```
|
||||
|
||||
| Flag | Effect |
|
||||
|---|---|
|
||||
| `--amlbi_pf_enabled True` | Flips PRS's ArgValidator gate so `type: uri_file` is accepted. |
|
||||
| `--amlbi_pf_run_mode component` | PF "I am a flow component" signal; PRS extracts the `output` field from each returned JSON string into the appended jsonl line. |
|
||||
| `--amlbi_file_format jsonl` | PRS parses the input file as jsonl and dispatches row dicts. Without it, `uri_file` → `list[file_path]` mini-batch (gotcha #9). |
|
||||
| `--amlbi_mini_batch_rows 1` | Switch from file-count to row-count batching. Pair with `--amlbi_file_format`. |
|
||||
|
||||
In `submit_pipeline.py`:
|
||||
|
||||
```python
|
||||
data_input = Input(
|
||||
path=str(HERE / "data" / "sample.jsonl"), # or azureml://datastore/.../*.jsonl
|
||||
type=AssetTypes.URI_FILE,
|
||||
mode="mount",
|
||||
)
|
||||
```
|
||||
|
||||
`mini_batch_size: "5"` then means 5 rows per mini-batch.
|
||||
`processor._iter_rows()` must handle the `list[dict]` shape (template
|
||||
already does — see the `isinstance(item, dict)` branch).
|
||||
|
||||
**Caveats of relying on this flag set:**
|
||||
|
||||
- The flags live in PRS's runtime arg parser but are **not part of the
|
||||
public schema**. They have worked unchanged across multiple PRS
|
||||
releases (PF used them in production), but Microsoft does not publish
|
||||
an SLA for them.
|
||||
- The output `parallel_run_step.jsonl` contains only the workflow output
|
||||
(extracted by PRS from each JSON line), not the full
|
||||
`{"line_number", "input", "output", "error"}` wrapper our `executor`
|
||||
returns. If downstream eval tooling needs the input echo or
|
||||
`line_number`, have `hooks.serialize_output` return a dict containing
|
||||
those fields.
|
||||
|
||||
**Alternative — file-list input (`uri_folder`):** keep `type: uri_folder`
|
||||
in both `submit_pipeline.py` and `component.yaml`, and **drop the
|
||||
`--amlbi_*` flags**. PRS will then dispatch each mini-batch as a
|
||||
`list[str]` of file paths (gotcha #9), so `mini_batch_size` becomes
|
||||
"files per batch" and `processor._iter_rows()` yields `{"path": ...}`
|
||||
rows. Use this only when each row really is its own opaque file (e.g.
|
||||
images, audio).
|
||||
|
||||
## 13. `EntryScriptException: No module named 'maf_prs'`
|
||||
|
||||
**Cause:** `component.yaml` uses `code: ./` + `entry_script: src/entry.py`
|
||||
(needed so `workflow.py` ships with the snapshot — see gotcha #3). PRS
|
||||
then uploads the project root and only puts the **project root** on
|
||||
`sys.path`. The entry module is loaded as `src.entry`, and inside it
|
||||
`from maf_prs.processor import create_processor` resolves `maf_prs` as a
|
||||
top-level package — which doesn't exist (it's actually `src.maf_prs`).
|
||||
Local dry-run masks this because we manually `sys.path.insert(0, 'src')`.
|
||||
|
||||
**Fix:** make `src/entry.py` add its own directory to `sys.path` **before**
|
||||
the first `maf_prs` import. The template asset already does this:
|
||||
|
||||
```python
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
if str(_HERE) not in sys.path:
|
||||
sys.path.insert(0, str(_HERE))
|
||||
|
||||
from maf_prs.processor import create_processor # noqa: E402
|
||||
```
|
||||
|
||||
Do **not** "fix" this by switching to `from .maf_prs.processor import
|
||||
create_processor` — PRS does not always load entry as part of a package,
|
||||
so the relative import will fail in some PRS runtime versions. The
|
||||
`sys.path` prepend is layout-agnostic and works in every PRS variant.
|
||||
|
||||
## 14. Image build aborts with `error: resolution-too-deep`
|
||||
|
||||
**Cause:** `env/conda.yml` lists pip dependencies without lower-bound
|
||||
version pins (e.g. bare `azureml-core`, `pandas`, `azure-identity`). Modern
|
||||
pip's resolver explores the version space exhaustively when there are no
|
||||
constraints; the AML build host hits its built-in
|
||||
`MAX_RESOLVER_DEPTH` and bails out before `conda env create` finishes.
|
||||
|
||||
The build log at `azureml-logs/20_image_build_log.txt` shows:
|
||||
|
||||
```
|
||||
DownloadPip subprocess error:
|
||||
error: resolution-too-deep
|
||||
|
||||
× Dependency resolution exceeded maximum depth
|
||||
╰─> Pip cannot resolve the current dependencies as the dependency graph is
|
||||
too complex for pip to solve efficiently.
|
||||
|
||||
CondaEnvException: Pip failed
|
||||
ERROR: failed to solve: process "/bin/sh -c ldconfig ... conda env create ..."
|
||||
did not complete successfully: exit code: 1
|
||||
```
|
||||
|
||||
The job never reaches `init()` — the failure is purely an environment
|
||||
build problem, even though the AML run UI lists it as a generic
|
||||
"Image build failed".
|
||||
|
||||
**Fix:** Pin every pip entry in `env/conda.yml` with a lower bound
|
||||
(`package>=X.Y.Z`). The skill's template `conda.yml` does this for every
|
||||
package; if you add a new dependency, add a lower bound for it too.
|
||||
|
||||
```yaml
|
||||
pip:
|
||||
- azureml-core>=1.59.0
|
||||
- azureml-mlflow>=1.59.0
|
||||
- azureml-dataset-runtime>=1.59.0
|
||||
- pandas>=2.2.0
|
||||
- agent-framework>=1.0.1
|
||||
- azure-identity>=1.19.0
|
||||
- azure-monitor-opentelemetry>=1.6.4
|
||||
```
|
||||
|
||||
Upper bounds are not needed and can cause integrity-check failures with
|
||||
the AML pre-installed packages — keep them open-ended.
|
||||
|
||||
## 15. `Optional input X must be placed in nested argument: $[[]]`
|
||||
|
||||
**Cause:** `component.yaml` declares an input as `optional: true` but
|
||||
references it with a bare `${{inputs.X}}` placeholder in
|
||||
`program_arguments`. AML refuses to register the component, with this
|
||||
exact error from `managementfrontend`:
|
||||
|
||||
```
|
||||
Error occurred when loading YAML file rootNode, details: Command "..."
|
||||
has error:
|
||||
Optional input X must be placed in nested argument: $[[]].
|
||||
```
|
||||
|
||||
Optional inputs need the **nested-argument syntax** so PRS can omit the
|
||||
flag entirely when the caller does not pass it (otherwise the flag would
|
||||
appear with an empty value, which most argparse-based scripts treat as a
|
||||
parse error).
|
||||
|
||||
**Fix:** Wrap the entire `--flag value` token in `$[[...]]`:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
api_version:
|
||||
type: string
|
||||
default: "2024-08-01-preview"
|
||||
optional: true
|
||||
|
||||
task:
|
||||
program_arguments: >-
|
||||
--model_endpoint ${{inputs.model_endpoint}}
|
||||
--model_deployment ${{inputs.model_deployment}}
|
||||
$[[--api_version ${{inputs.api_version}}]]
|
||||
--output_dir ${{outputs.debug_info}}
|
||||
```
|
||||
|
||||
Required inputs (no `optional: true`) keep the bare `${{inputs.X}}`
|
||||
form — only optional inputs need `$[[...]]`. If you would rather avoid
|
||||
the special syntax, drop `optional: true` from the input declaration —
|
||||
a `default:` is enough to make callers' lives easy without making the
|
||||
input optional at the schema level.
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
# Prompt Flow PRS vs. MAF PRS — Side-by-Side Mapping
|
||||
|
||||
Use this table during Phase 1 audit to translate each piece of the existing
|
||||
PF PRS submission into the MAF equivalent.
|
||||
|
||||
## 0. Background — what `load_component(flow.dag.yaml)` did automatically
|
||||
|
||||
In the Prompt Flow PRS pattern (see
|
||||
[examples/tutorials/run-flow-with-pipeline/pipeline.ipynb](../../../../examples/tutorials/run-flow-with-pipeline/pipeline.ipynb)),
|
||||
`load_component("flow.dag.yaml")` produced an Azure ML **parallel component**
|
||||
with the following pieces filled in for free:
|
||||
|
||||
| Auto-generated piece | Where it came from |
|
||||
|---|---|
|
||||
| Input port `data` (`uri_file` / `uri_folder`) | Implicit from PRS |
|
||||
| Output port `flow_outputs` (`uri_file` → `parallel_run_step.jsonl`) | Implicit from PRS |
|
||||
| Output port `debug_info` (`uri_folder`) | Implicit from PRS |
|
||||
| Component parameters (flow inputs + connections) | Parsed from `flow.dag.yaml` |
|
||||
| Environment | Inherited from latest promptflow runtime image |
|
||||
| Entry script (`init`/`run`) | Generated by promptflow runtime |
|
||||
| Column mapping (`url="${data.url}"`) | Driven by flow input names |
|
||||
|
||||
MAF has **no equivalent auto-conversion**. The skill produces the same
|
||||
five artefacts by hand (entry script + processor/executor + component YAML
|
||||
+ conda env + submission script).
|
||||
|
||||
## 1. Component artefacts
|
||||
|
||||
| Concern | Prompt Flow PRS | MAF PRS (this skill) |
|
||||
|---|---|---|
|
||||
| Component definition | Auto-generated by `load_component("flow.dag.yaml")` | Hand-written `component.yaml` (`type: parallel`) |
|
||||
| Entry script | Provided by promptflow runtime | `src/entry.py` — thin wrapper exposing `init()` / `run(mini_batch, context)` / `shutdown()` |
|
||||
| Plumbing layer | `promptflow.parallel` (`AbstractParallelRunProcessor` + `ComponentRunExecutor`) | `src/maf_prs/{processor,executor,config}.py` (mirrors the same split) |
|
||||
| Environment | Inherited from latest promptflow runtime image | `env/conda.yml` declared in the component |
|
||||
| Component parameters | Auto-derived from flow inputs + `connections` | Declared explicitly under `inputs:` in `component.yaml` |
|
||||
| Input port type | PF accepted `uri_file` directly (runtime emitted the `--amlbi_pf_*` flag set automatically) | Vanilla PRS rejects `uri_file` unless `program_arguments` carries the same PF compatibility flag set (`--amlbi_pf_enabled True --amlbi_pf_run_mode component --amlbi_file_format jsonl --amlbi_mini_batch_rows 1`). Default for this skill (gotcha #12). |
|
||||
| Output ports | `flow_outputs` (jsonl), `debug_info` (folder) | Same names, declared explicitly |
|
||||
| Append rule | `parallel_run_step.jsonl` | `append_row_to: ${{outputs.flow_outputs}}` |
|
||||
|
||||
## 2. Per-row data binding
|
||||
|
||||
| Concern | Prompt Flow PRS | MAF PRS |
|
||||
|---|---|---|
|
||||
| Column → input mapping | `flow_node(url="${data.url}")` declared in pipeline DSL; resolved at runtime by `FlowExecutor.apply_inputs_mapping` | Pure Python: `hooks.build_workflow_input(row)` reads `row["url"]` and returns whatever the workflow's first executor expects |
|
||||
| Where the mapping lives | `submit_pipeline.py` (declarative) | `src/hooks.py` (imperative) — the only file most users edit |
|
||||
| Input format | `PF_INPUT_FORMAT` env var | `processor.py` uses `pandas`; jsonl/csv/tsv all work without an env var |
|
||||
| File-mode input (`uri_folder` of opaque files) | PF iterates files of allowed extensions | `processor._iter_rows()` yields `{"path": ...}` per file |
|
||||
| Stable row id | `Row.from_dict(data, row_number=base+idx)` where `base = context.global_row_index_lower_bound` | Same: `processor.process()` reads `context.global_row_index_lower_bound` and stamps `line_number` on each result |
|
||||
|
||||
## 3. Connections / secrets
|
||||
|
||||
| Concern | Prompt Flow PRS | MAF PRS |
|
||||
|---|---|---|
|
||||
| Endpoint URL + deployment | `connections={"node": {"connection": "...", "deployment_name": "..."}}` | Component `inputs:` (e.g. `model_endpoint`, `model_deployment`) wired through `program_arguments` to env vars |
|
||||
| API key | Stored in PF connection | Prefer **Managed Identity + Key Vault**; if a key is unavoidable, inject as a workspace secret env var on the deployment, never in YAML |
|
||||
| Multiple LLM nodes with different connections | Per-node `connections` map | One set of inputs per distinct chat client; usually a single `(endpoint, deployment)` pair suffices because the workflow already encapsulates routing |
|
||||
|
||||
## 4. PRS run settings (carry over verbatim)
|
||||
|
||||
These have a 1:1 mapping — copy the values from the original PF script
|
||||
unchanged unless the user explicitly wants to tune.
|
||||
|
||||
| PF (`flow_node.*`) | MAF (`component.yaml` and `pipeline_node.*`) |
|
||||
|---|---|
|
||||
| `compute = "cpu-cluster"` | `pipeline_node.compute = "cpu-cluster"` |
|
||||
| `resources = {"instance_count": N}` | `pipeline_node.resources = {"instance_count": N}` |
|
||||
| `mini_batch_size = K` | `pipeline_node.mini_batch_size = K` (and default in `component.yaml`) |
|
||||
| `max_concurrency_per_instance = M` | same |
|
||||
| `retry_settings = {"max_retries": ..., "timeout": ...}` | same |
|
||||
| `error_threshold = -1` | same |
|
||||
| `mini_batch_error_threshold = -1` | same |
|
||||
| `logging_level = "DEBUG"` | same |
|
||||
| `environment_variables = {"PF_INPUT_FORMAT": "jsonl"}` | Pass via `program_arguments` or `pipeline_node.environment_variables` |
|
||||
| `outputs.flow_outputs.mode = "mount"` (when `instance_count > 1`) | same — required, not optional |
|
||||
| `outputs.debug_info.mode = "mount"` (when `instance_count > 1`) | same |
|
||||
|
||||
## 5. Pipeline DSL
|
||||
|
||||
| Concern | Prompt Flow PRS | MAF PRS |
|
||||
|---|---|---|
|
||||
| Imports | `from azure.ai.ml import load_component, MLClient, Input, Output` | Same |
|
||||
| `@pipeline()` decorator | `from azure.ai.ml.dsl import pipeline` | Same |
|
||||
| Column-mapping arguments | `flow_node(url="${data.url}", question="${data.q}")` | **Not present** — mapping moved into Python (`executor.build_workflow_input`); the pipeline call only passes `data` + connection inputs |
|
||||
| Submission | `ml_client.jobs.create_or_update(pipeline_job_def, experiment_name=...)` | Same |
|
||||
| Streaming | `ml_client.jobs.stream(...)` | Same |
|
||||
| Scheduler / batch endpoint (notebook §4) | works on PF flow_component | works unchanged on the MAF parallel component — no special handling needed |
|
||||
|
||||
## 6. Things PF did automatically that you must do explicitly
|
||||
|
||||
| Auto-done by PF | You must do this in MAF |
|
||||
|---|---|
|
||||
| Run a forked Python process per worker that hosts the flow runtime | `entry.py` exposes `init()` / `run(mini_batch, context)` / `shutdown()`; `processor.init()` builds a workflow factory + reusable event loop |
|
||||
| Convert each row into a flow run | `executor.execute(row, row_number)` builds the input via `hooks.build_workflow_input(row)` and `await`s `workflow.run(...)` |
|
||||
| Apply `${data.col}` template mapping | Edit `hooks.build_workflow_input(row)` in `src/hooks.py` — plain Python, no template engine. The skill agent fills this automatically when the source PF mapping is parseable **and** the workflow's start handler input is typed (see SKILL.md Phase 1.5 checks A–D); otherwise it leaves a `# TODO` stub naming the missing piece. |
|
||||
| Append `parallel_run_step.jsonl` lines | `processor.process(...)` returns a `list[str]` from JSON-serialised dicts; PRS appends each line |
|
||||
| Surface `debug_info` automatically | Decide what (if anything) to write to `--output_dir` in `hooks.setup` / a custom `executor.finalize` |
|
||||
| Validate input/output ports | Validate manually by running `entry.py` against the local sample before submitting |
|
||||
| Aggregation node finalize (`AggregationFinalizer`) | Override `executor.finalize()` to consume the temp jsonl rows if you need a global reduce step |
|
||||
@@ -0,0 +1,364 @@
|
||||
---
|
||||
name: maf-tracing
|
||||
description: "Enable tracing and logging for Microsoft Agent Framework (MAF) workflows. Configures OpenTelemetry export to Azure Application Insights and/or a generic OTLP endpoint using environment variables. Adds required packages to requirements.txt. WHEN: enable tracing, add tracing, enable logging, add logging, configure telemetry, Application Insights for MAF, OTLP export, observe workflow, monitor agent workflow, trace agent framework, instrument MAF, add observability, trace workflow executions, debug workflow."
|
||||
license: MIT
|
||||
metadata:
|
||||
author: Team
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
# Enable Tracing & Logging for MAF Workflows
|
||||
|
||||
> Configure OpenTelemetry-based tracing for Microsoft Agent Framework workflows, exporting to Azure Application Insights and/or a generic OTLP endpoint.
|
||||
|
||||
## Triggers
|
||||
|
||||
Activate this skill when the user wants to:
|
||||
- Enable tracing or logging for a MAF workflow
|
||||
- Export telemetry to Azure Application Insights
|
||||
- Export traces to an OTLP-compatible endpoint (Jaeger, Zipkin, Grafana Tempo, etc.)
|
||||
- Add observability or monitoring to an `agent-framework` project
|
||||
- Debug or inspect executor/agent/LLM spans in a MAF workflow
|
||||
|
||||
## Background
|
||||
|
||||
MAF automatically emits OpenTelemetry spans for every executor invocation, agent call, and LLM request. No instrumentation changes are needed inside `Executor` classes. You only need to:
|
||||
|
||||
1. **Configure an exporter** — where spans are sent (Application Insights, OTLP endpoint, or both)
|
||||
2. **Call `configure_otel_providers()`** — activates MAF's built-in instrumentation
|
||||
|
||||
This must happen **once at application startup**, before any `workflow.run()` calls.
|
||||
|
||||
---
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Read the user's project first** — Check for an existing `requirements.txt`, `.env`, and entry-point script (e.g., `main.py`, `app.py`, `run_*.py`).
|
||||
2. **Ask what export destination(s) are needed** — Application Insights, OTLP endpoint, or both. If unclear, default to both.
|
||||
3. **Do not modify Executor classes** — MAF tracing is automatic. Never add manual `tracer.start_span()` calls inside `@handler` methods unless the user explicitly asks for custom spans.
|
||||
4. **Call order matters** — Exporter configuration must happen BEFORE `configure_otel_providers()`, and both must happen BEFORE any `workflow.run()`.
|
||||
5. **Add packages to `requirements.txt`** — Append only the packages the user needs (see Packages section). Do not duplicate existing entries.
|
||||
6. **Use environment variables** — Never hardcode connection strings or endpoints. Always read from `os.environ` or `.env`.
|
||||
7. **Generate `.env.example`** — Provide a template showing which environment variables are required.
|
||||
8. **Python logging integration** — When the user asks for logging (not just tracing), also configure the Python `logging` module to export via OpenTelemetry using `opentelemetry-sdk` log handler.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Required For | Description |
|
||||
|----------|-------------|-------------|
|
||||
| `APPLICATIONINSIGHTS_CONNECTION_STRING` | Application Insights | Connection string from Azure Portal → App Insights → Overview |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP export | Base URL of the OTLP collector (e.g., `http://localhost:4318`) |
|
||||
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | OTLP export (traces only) | Overrides the base endpoint for trace signals only |
|
||||
| `OTEL_EXPORTER_OTLP_PROTOCOL` | OTLP export | Protocol: `http/protobuf` (default) or `grpc` |
|
||||
| `OTEL_SERVICE_NAME` | Optional | Service name shown in trace backends (defaults to Python process name) |
|
||||
|
||||
> `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT` for traces.
|
||||
|
||||
---
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Version | When Needed |
|
||||
|---------|---------|-------------|
|
||||
| `agent-framework` | >=1.0.1 | Always (provides `configure_otel_providers`) |
|
||||
| `azure-monitor-opentelemetry` | >=1.6.4 | Application Insights export |
|
||||
| `opentelemetry-exporter-otlp-proto-http` | >=1.25.0 | OTLP/HTTP export |
|
||||
| `opentelemetry-exporter-otlp-proto-grpc` | >=1.25.0 | OTLP/gRPC export (only if `OTEL_EXPORTER_OTLP_PROTOCOL=grpc`) |
|
||||
| `opentelemetry-sdk` | >=1.25.0 | Custom spans or Python logging integration |
|
||||
| `python-dotenv` | any | Loading `.env` files |
|
||||
|
||||
---
|
||||
|
||||
## Setup Patterns
|
||||
|
||||
### Pattern A: Application Insights Only
|
||||
|
||||
Use when the user wants to send traces to Azure Application Insights.
|
||||
|
||||
**Required env var:** `APPLICATIONINSIGHTS_CONNECTION_STRING`
|
||||
|
||||
**Required packages:**
|
||||
```
|
||||
azure-monitor-opentelemetry>=1.6.4
|
||||
```
|
||||
|
||||
**Setup code** (add at the top of the entry-point script, before `workflow.run()`):
|
||||
|
||||
```python
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Step 1: Configure Azure Monitor exporter (traces, metrics, logs → App Insights)
|
||||
configure_azure_monitor(
|
||||
connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
|
||||
)
|
||||
|
||||
# Step 2: Enable MAF's built-in instrumentation (executor, agent, LLM spans)
|
||||
configure_otel_providers()
|
||||
```
|
||||
|
||||
### Pattern B: OTLP Endpoint Only
|
||||
|
||||
Use when the user wants to send traces to a generic OTLP-compatible backend (Jaeger, Grafana Tempo, Aspire Dashboard, etc.).
|
||||
|
||||
**Required env var:** `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||||
|
||||
**Required packages:**
|
||||
```
|
||||
opentelemetry-sdk>=1.25.0
|
||||
opentelemetry-exporter-otlp-proto-http>=1.25.0
|
||||
```
|
||||
|
||||
**Setup code:**
|
||||
|
||||
```python
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Step 1: Set up the OTLP exporter with a TracerProvider
|
||||
resource = Resource.create({
|
||||
"service.name": os.environ.get("OTEL_SERVICE_NAME", "maf-workflow"),
|
||||
})
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
|
||||
otlp_exporter = OTLPSpanExporter(
|
||||
endpoint=os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
||||
)
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
# Step 2: Enable MAF's built-in instrumentation
|
||||
configure_otel_providers()
|
||||
```
|
||||
|
||||
### Pattern C: Both Application Insights and OTLP
|
||||
|
||||
Use when the user wants dual export — Application Insights for Azure-native monitoring plus an OTLP backend for local/third-party observability.
|
||||
|
||||
**Required env vars:** `APPLICATIONINSIGHTS_CONNECTION_STRING`, `OTEL_EXPORTER_OTLP_ENDPOINT`
|
||||
|
||||
**Required packages:**
|
||||
```
|
||||
azure-monitor-opentelemetry>=1.6.4
|
||||
opentelemetry-sdk>=1.25.0
|
||||
opentelemetry-exporter-otlp-proto-http>=1.25.0
|
||||
```
|
||||
|
||||
**Setup code:**
|
||||
|
||||
```python
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Step 1a: Configure Azure Monitor (sets up its own TracerProvider internally)
|
||||
configure_azure_monitor(
|
||||
connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
|
||||
)
|
||||
|
||||
# Step 1b: Add OTLP exporter to the existing TracerProvider
|
||||
otlp_endpoint = (
|
||||
os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
)
|
||||
if otlp_endpoint:
|
||||
otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
|
||||
tracer_provider: TracerProvider = trace.get_tracer_provider()
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
|
||||
|
||||
# Step 2: Enable MAF's built-in instrumentation
|
||||
configure_otel_providers()
|
||||
```
|
||||
|
||||
### Pattern D: Python Logging via OpenTelemetry
|
||||
|
||||
Use when the user also wants Python `logging` calls to be exported alongside traces.
|
||||
|
||||
**Additional packages (on top of Pattern A, B, or C):**
|
||||
```
|
||||
opentelemetry-sdk>=1.25.0
|
||||
```
|
||||
|
||||
**Setup code** (add after the exporter setup, before `configure_otel_providers()`):
|
||||
|
||||
```python
|
||||
import logging
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
|
||||
# If using Application Insights, configure_azure_monitor() already handles log export.
|
||||
# If using OTLP only, set up the OTLP log exporter:
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(
|
||||
endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"),
|
||||
))
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
# Bridge Python logging to OpenTelemetry
|
||||
from opentelemetry.instrumentation.logging import LoggingInstrumentor
|
||||
LoggingInstrumentor().instrument(set_logging_format=True)
|
||||
```
|
||||
|
||||
> **Note:** When using `azure-monitor-opentelemetry` (Pattern A/C), `configure_azure_monitor()` already captures Python logs by default. The above is only needed for OTLP-only setups.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
### Step 1: Determine export destination(s)
|
||||
|
||||
Ask the user or infer from context:
|
||||
- Application Insights → Pattern A
|
||||
- OTLP endpoint → Pattern B
|
||||
- Both → Pattern C
|
||||
|
||||
If the user says "tracing" without specifying a destination, default to Pattern C (both).
|
||||
|
||||
### Step 2: Update `requirements.txt`
|
||||
|
||||
Append the required packages. Do **not** duplicate existing entries. Example additions for Pattern C:
|
||||
|
||||
```
|
||||
azure-monitor-opentelemetry>=1.6.4
|
||||
opentelemetry-sdk>=1.25.0
|
||||
opentelemetry-exporter-otlp-proto-http>=1.25.0
|
||||
```
|
||||
|
||||
### Step 3: Create or update `.env.example`
|
||||
|
||||
Add the environment variables relevant to the chosen pattern:
|
||||
|
||||
```bash
|
||||
# === Tracing & Observability ===
|
||||
# Application Insights (Azure Portal → App Insights → Overview → Connection String)
|
||||
APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=xxx;IngestionEndpoint=https://xxx.in.applicationinsights.azure.com/
|
||||
|
||||
# OTLP endpoint (e.g., Jaeger, Grafana Tempo, Aspire Dashboard)
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
|
||||
|
||||
# Optional: override service name in trace backends
|
||||
OTEL_SERVICE_NAME=my-maf-workflow
|
||||
```
|
||||
|
||||
### Step 4: Add tracing setup to entry-point script
|
||||
|
||||
Insert the setup code from the appropriate pattern at the **top** of the entry-point script (after imports, before any `workflow.run()` call). The setup must execute once at module load / application startup.
|
||||
|
||||
**Placement rules:**
|
||||
- If the entry point uses `asyncio.run(main())`, place setup code inside `main()` before `workflow.run()`.
|
||||
- If the entry point is a module-level script, place setup code after imports and `load_dotenv()`.
|
||||
- If the entry point is a web server (FastAPI, Flask), place setup code in the application factory or startup event.
|
||||
|
||||
### Step 5: Verify
|
||||
|
||||
1. Run the workflow and check that traces appear in the configured destination.
|
||||
2. For Application Insights: Azure Portal → Application Insights → Transaction Search → filter by "Dependency" or "Request".
|
||||
3. For OTLP: Check the collector backend UI (e.g., Jaeger UI at `http://localhost:16686`).
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
1. **Call order matters** — `configure_azure_monitor()` and/or OTLP exporter setup must happen BEFORE `configure_otel_providers()`. If reversed, MAF spans won't be exported.
|
||||
2. **`configure_otel_providers()` must run BEFORE `workflow.run()`** — Otherwise, executor-level spans are not generated.
|
||||
3. **Do not call setup code inside Executors** — Tracing setup is application-level, not per-request. Calling it inside a `@handler` method will create duplicate exporters and corrupt traces.
|
||||
4. **`configure_azure_monitor()` creates its own TracerProvider** — When combining with OTLP (Pattern C), add the OTLP exporter to the existing provider via `trace.get_tracer_provider()` rather than creating a new `TracerProvider`.
|
||||
5. **Missing `configure_otel_providers()`** — Without this call, you'll see Application Insights or OTLP infrastructure telemetry but no MAF-specific spans (executor transitions, agent calls, LLM requests).
|
||||
6. **Connection string format** — The `APPLICATIONINSIGHTS_CONNECTION_STRING` starts with `InstrumentationKey=` followed by a GUID. Do not confuse it with the Instrumentation Key alone.
|
||||
7. **OTLP endpoint trailing path** — `OTEL_EXPORTER_OTLP_ENDPOINT` should be the base URL (e.g., `http://localhost:4318`). The SDK appends `/v1/traces` automatically. Do not include `/v1/traces` in the env var.
|
||||
8. **gRPC vs HTTP** — Default protocol is `http/protobuf` (port 4318). If the collector uses gRPC (port 4317), set `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` and install `opentelemetry-exporter-otlp-proto-grpc` instead.
|
||||
|
||||
---
|
||||
|
||||
## Example: Complete Entry Point with Tracing
|
||||
|
||||
```python
|
||||
"""Entry point for a MAF workflow with full tracing setup."""
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
def setup_tracing():
|
||||
"""Configure telemetry exporters and MAF instrumentation. Call once at startup."""
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
# Application Insights
|
||||
appinsights_conn = os.environ.get("APPLICATIONINSIGHTS_CONNECTION_STRING")
|
||||
if appinsights_conn:
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
configure_azure_monitor(connection_string=appinsights_conn)
|
||||
|
||||
# OTLP endpoint (optional, additive)
|
||||
otlp_endpoint = (
|
||||
os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
)
|
||||
if otlp_endpoint:
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
|
||||
otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
|
||||
tracer_provider = trace.get_tracer_provider()
|
||||
|
||||
# If Azure Monitor already set a TracerProvider, reuse it; otherwise create one
|
||||
if not isinstance(tracer_provider, TracerProvider):
|
||||
resource = Resource.create({
|
||||
"service.name": os.environ.get("OTEL_SERVICE_NAME", "maf-workflow"),
|
||||
})
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
|
||||
|
||||
# Enable MAF's built-in spans (must be last)
|
||||
configure_otel_providers()
|
||||
|
||||
|
||||
async def main():
|
||||
setup_tracing()
|
||||
|
||||
# Import and run your workflow here
|
||||
from workflow import create_workflow
|
||||
|
||||
workflow = create_workflow()
|
||||
result = await workflow.run("Hello, world!")
|
||||
print(result.get_outputs())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
@@ -0,0 +1,160 @@
|
||||
---
|
||||
name: promptflow-to-maf
|
||||
description: "Convert Prompt Flow flow definitions to Microsoft Agent Framework (MAF) workflows. Parses flow.dag.yaml, maps nodes to Executors, and generates runnable Python code using agent-framework 1.0.x. WHEN: convert promptflow, migrate promptflow, promptflow to MAF, promptflow to agent framework, convert flow.dag.yaml, migrate flow to MAF, convert PF flow, PF to agent-framework, convert DAG flow to workflow, migrate LLM flow. DO NOT USE FOR: writing new MAF workflows from scratch (no source flow), deploying MAF workflows (use maf-online-endpoint), enabling tracing (use maf-tracing), or general agent-framework Q&A."
|
||||
license: MIT
|
||||
metadata:
|
||||
author: Team
|
||||
version: "2.0.0"
|
||||
---
|
||||
|
||||
# Prompt Flow → Microsoft Agent Framework Conversion
|
||||
|
||||
> Convert Prompt Flow `flow.dag.yaml` definitions into runnable MAF `WorkflowBuilder` Python code.
|
||||
|
||||
## Triggers
|
||||
|
||||
Activate this skill when the user wants to:
|
||||
- Convert a Prompt Flow flow to Microsoft Agent Framework
|
||||
- Migrate a `flow.dag.yaml` to MAF workflow code
|
||||
- Rebuild a Prompt Flow application using `agent-framework`
|
||||
|
||||
---
|
||||
|
||||
## What to Read When (Progressive Disclosure)
|
||||
|
||||
This skill is split across multiple files. **Always read this file first.** Then read additional files based on what the source flow contains:
|
||||
|
||||
| Situation | Required Reading |
|
||||
|---|---|
|
||||
| **Every conversion task** | This file + [references/gotchas.md](references/gotchas.md) |
|
||||
| Need to map a specific node type | [references/node-mapping.md](references/node-mapping.md) |
|
||||
| Writing Executor handlers / picking LLM client / setting `temperature`/`max_tokens` | [references/workflow-context.md](references/workflow-context.md) |
|
||||
| Source flow has a node with `source.type: package` | [topics/custom-tool-nodes.md](topics/custom-tool-nodes.md) |
|
||||
| Source flow has image / multimodal inputs | [topics/multimodal.md](topics/multimodal.md) + [examples/multimodal-chat.md](examples/multimodal-chat.md) |
|
||||
| Source flow has any node with `aggregation: true` | [topics/evaluation-flows.md](topics/evaluation-flows.md) + [templates/eval_runner.py](templates/eval_runner.py) + [examples/evaluation.md](examples/evaluation.md) |
|
||||
| Want a complete reference example | [examples/linear-chat.md](examples/linear-chat.md) (basic), [examples/multimodal-chat.md](examples/multimodal-chat.md), [examples/evaluation.md](examples/evaluation.md) |
|
||||
|
||||
> **Don't pre-load everything.** Read each file lazily when its situation is detected during Phase 1 audit.
|
||||
|
||||
---
|
||||
|
||||
## Core Rules (apply to every conversion)
|
||||
|
||||
1. **Read the source flow first** — Always parse `flow.dag.yaml`, all referenced source files (`.jinja2`, `.py`), and `requirements.txt` before generating anything.
|
||||
2. **Preserve prompts verbatim** — System prompts, user prompt templates, and any text from `.jinja2` or inline prompt nodes must be copied exactly as they appear in the original Prompt Flow. Do not rephrase, summarize, add, or remove any content — including examples, instructions, formatting, and preambles (e.g., "Read the following conversation and respond:"). The MAF workflow must send the identical prompt text to the LLM.
|
||||
3. **One Executor per node** — Each Prompt Flow node becomes one `Executor` subclass with a `@handler` method. (Some node combinations may be safely merged — see [references/node-mapping.md](references/node-mapping.md) for "Node Collapsing Patterns".)
|
||||
4. **Preserve behaviour** — The MAF workflow must produce the same outputs for the same inputs as the original flow.
|
||||
5. **Use GA packages** — `agent-framework>=1.0.1`, `agent-framework-openai>=1.0.1`. Use preview packages (`--pre`) only for orchestrations, Azure AI Search, or multi-agent features. (Full table in [references/workflow-context.md](references/workflow-context.md).)
|
||||
6. **Create output folder** — Place generated files in a sibling folder named `<original-folder>-maf/`.
|
||||
7. **Copy user-defined Python packages** — If the flow imports from internal packages (e.g., `my_utils/`, helper modules), copy the entire package directory into the output folder. The MAF workflow imports directly from the local copy — no `sys.path` manipulation needed.
|
||||
8. **Generate a test sample** — Always include a runnable `test_<name>.py` sample script.
|
||||
9. **Never modify the original flow** — All output goes into the new folder.
|
||||
10. **Evaluation flows use the EvalRunner pattern** — If any node has `aggregation: true`, the flow is an evaluation flow. See [topics/evaluation-flows.md](topics/evaluation-flows.md).
|
||||
11. **Always export a `create_workflow()` factory** — MAF workflows do not support concurrent `run()` calls on a single instance (`RuntimeError: Workflow is already running`). Every generated `workflow.py` must export a `create_workflow()` factory function that creates a fresh workflow instance per call. Do NOT instantiate Executors or build the workflow at module level. This ensures callers can safely run multiple workflows concurrently (e.g., evaluation batches, parallel API requests, or test suites). For evaluation flows, `EvalRunner` relies on this factory to create one workflow per row.
|
||||
12. **Copy ALL referenced resources into the output folder** — The generated `-maf/` project must be fully self-contained with zero dependencies on the original Prompt Flow folder. Copy every resource file the flow references:
|
||||
- **Data files** (`.jsonl`, `.csv`, `.json`, `.tsv`) used for testing or evaluation
|
||||
- **Prompt / template files** (`.jinja2`, `.md` used as prompts)
|
||||
- **User-defined Python modules** (`.py` files or packages imported by nodes — see rule 7)
|
||||
- **Any other non-code assets** (e.g., `samples.json`, config files, image assets)
|
||||
|
||||
Update all file path references (e.g., `DEFAULT_DATA`, `_TEMPLATES_DIR`, `_PROMPT_TEMPLATE`) to point to the local copy using `Path(__file__).parent / ...`. Never use `parent.parent` or relative paths that reach back into the original flow directory.
|
||||
13. **Preserve graph topology and conditions exactly** — The MAF workflow's graph structure MUST be equivalent to the original `flow.dag.yaml` graph. Specifically:
|
||||
- **Node coverage** — Every Prompt Flow node must map to exactly one MAF Executor (or be merged via an explicitly allowed Collapsing Pattern; see [references/node-mapping.md](references/node-mapping.md)). No PF node may be silently dropped, and no extra Executors may be invented that don't correspond to a PF node or an allowed merge.
|
||||
- **Edge coverage** — Every data reference `${node.output}` in `flow.dag.yaml` must correspond to a MAF edge (`add_edge` / `add_fan_out_edges` / `add_fan_in_edges`) connecting the equivalent Executors. No edges may be added or removed.
|
||||
- **Parallelism preserved** — If two PF nodes run in parallel from a shared upstream node, they must remain parallel in MAF (`add_fan_out_edges`). Do NOT serialize parallel branches. If multiple PF nodes fan into one downstream node, they must use `add_fan_in_edges`.
|
||||
- **Conditions preserved** — Every `activate_config` (when/is) in PF must become an `add_edge(..., condition=fn)` with semantically identical predicate logic. The truth value of the condition for any given input must match the original.
|
||||
- **No reordering** — The execution order implied by the dependency graph must be preserved. Do not move logic from a downstream node into an upstream node (or vice versa) in a way that changes when work happens relative to other branches.
|
||||
- **Mapping table required** — In Phase 1, produce an explicit PF-node → MAF-Executor / edge mapping table (see Phase 1 step 6) and verify it in Phase 4 (see Phase 4 step 22). Any allowed merge must be annotated with the matching Collapsing Pattern from [references/node-mapping.md](references/node-mapping.md).
|
||||
|
||||
---
|
||||
|
||||
## Conversion Workflow (4 Phases)
|
||||
|
||||
### Phase 1 — Audit the Prompt Flow
|
||||
|
||||
1. **Read `flow.dag.yaml`** — identify all inputs, outputs, nodes, their types, and edges (data references like `${node.output}`).
|
||||
- For every node, record `type` AND `source.type`. **A node with `source.type: package` is a custom user-defined tool — read [topics/custom-tool-nodes.md](topics/custom-tool-nodes.md) and call it directly from the Executor; do NOT remap to `OpenAIChatClient`/`Agent`.**
|
||||
2. **Read source files** — open every `.jinja2` template, every `.py` file referenced by `source.type: code` nodes, and the package source for every `source.type: package` node.
|
||||
3. **Read `requirements.txt`** — note any extra dependencies.
|
||||
4. **Map the graph** — draw the node dependency graph from `${...}` references. Identify:
|
||||
- Linear chains (A → B → C)
|
||||
- Parallel branches (A → B, A → C)
|
||||
- Conditional branches (`activate_config`)
|
||||
- Fan-in / aggregation points
|
||||
5. **Detect special cases — load the matching topic file:**
|
||||
- Any node with `aggregation: true` → evaluation flow → load [topics/evaluation-flows.md](topics/evaluation-flows.md)
|
||||
- Any node with `source.type: package` → custom tool → load [topics/custom-tool-nodes.md](topics/custom-tool-nodes.md)
|
||||
- Any image inputs (dict with `data:image/*;url` key, or string starting with `data:image/`) → multimodal → load [topics/multimodal.md](topics/multimodal.md)
|
||||
6. **Produce a node-mapping table** — Before writing any MAF code, emit (in your reasoning or as a comment block at the top of `workflow.py`) an explicit table that lists, for every PF node:
|
||||
- PF node name and `type` (+ `source.type`)
|
||||
- The MAF Executor it maps to (or the merged Executor name, with the matching Collapsing Pattern from [references/node-mapping.md](references/node-mapping.md))
|
||||
- The incoming edges (PF `${...}` references → MAF `add_edge` / `add_fan_in_edges`)
|
||||
- The outgoing edges (PF downstream consumers → MAF `add_edge` / `add_fan_out_edges`)
|
||||
- Any `activate_config` → the MAF `condition=fn` it becomes
|
||||
|
||||
This table is the contract used to verify graph equivalence in Phase 4. Every PF node must appear; every `${...}` reference must appear as an edge.
|
||||
|
||||
### Phase 2 — Generate MAF Code
|
||||
|
||||
7. **Create output folder** — `<original-folder>-maf/`.
|
||||
8. **Copy internal packages** — see Rule 7 above.
|
||||
9. **Copy all referenced resources** — see Rule 12 above.
|
||||
10. **Create one Executor per node** following the mapping table from Phase 1 step 6 and [references/node-mapping.md](references/node-mapping.md). Do not invent extra Executors and do not silently merge nodes outside of the explicitly allowed Collapsing Patterns.
|
||||
11. **Wire the workflow inside a `create_workflow()` factory function** using `WorkflowBuilder`. The edges you add MUST exactly match the edges listed in the Phase 1 mapping table. Executor instantiation and `WorkflowBuilder.build()` must happen inside this function — not at module level — so each call returns a fresh, independent workflow instance:
|
||||
- `.add_edge(source, target)` for linear connections
|
||||
- `.add_edge(source, target, condition=fn)` for conditionals (one per PF `activate_config`, with semantically identical predicate)
|
||||
- `.add_fan_out_edges(source, [targets])` for parallel branches (preserve PF parallelism — never serialize)
|
||||
- `.add_fan_in_edges([sources], target)` for aggregation
|
||||
12. **Handle LLM nodes**:
|
||||
- Extract system prompt from `.jinja2` template → `Agent(instructions="...")`
|
||||
- Pick the right client — see [references/workflow-context.md](references/workflow-context.md)
|
||||
- `Agent.run()` returns an `AgentResponse` — extract text with `.text`
|
||||
- **Preserve LLM parameters** — pass `temperature`, `max_tokens`, etc. via `OpenAIChatOptions` (see [references/workflow-context.md](references/workflow-context.md))
|
||||
13. **Handle chat history** — format prior turns into a prompt string in an InputExecutor, not as raw message dicts.
|
||||
14. **Handle Python tool nodes** — convert to plain functions and pass to `Agent(tools=[fn])`.
|
||||
15. **For evaluation flows / multimodal flows / custom-tool nodes** — follow the topic file you loaded in Phase 1 step 5.
|
||||
|
||||
### Phase 3 — Generate Supporting Files
|
||||
|
||||
16. **`requirements.txt`** — include only needed `agent-framework-*` packages. Add `azure-identity>=1.15.0` if any LLM client uses the identity template.
|
||||
17. **`.env.example`** — template with required environment variables (endpoint, model, key only if the connection uses key auth).
|
||||
18. **`test_<name>.py`** — runnable sample script exercising single-turn and multi-turn (if applicable).
|
||||
19. **`README.md`** — brief setup and run instructions. (Other documentation only if the user requests it.)
|
||||
|
||||
### Phase 4 — Validate
|
||||
|
||||
20. **Create a virtual environment** and install dependencies.
|
||||
21. **Run the test sample** to verify the workflow produces output.
|
||||
22. **Verify graph topology equivalence against `flow.dag.yaml`** — re-open the source `flow.dag.yaml` and the Phase 1 mapping table, then check:
|
||||
- [ ] Every PF node appears in the mapping table and is realized as exactly one MAF Executor (or is part of an explicitly annotated Collapsing Pattern).
|
||||
- [ ] No MAF Executor exists that does not correspond to a PF node or an annotated merge.
|
||||
- [ ] Every PF `${node.output}` reference is realized as a MAF edge between the corresponding Executors.
|
||||
- [ ] No MAF edges exist that are not present in PF.
|
||||
- [ ] PF parallel branches use `add_fan_out_edges`; PF fan-in points use `add_fan_in_edges`. No parallel branch has been serialized.
|
||||
- [ ] Every PF `activate_config` has a matching `add_edge(..., condition=fn)` whose predicate is semantically identical (same truth value for the same inputs).
|
||||
|
||||
If any check fails, fix the workflow before proceeding.
|
||||
23. **Fix errors** — see [references/gotchas.md](references/gotchas.md).
|
||||
|
||||
---
|
||||
|
||||
## Skill File Index
|
||||
|
||||
```
|
||||
.github/skills/promptflow-to-maf/
|
||||
├── SKILL.md ← This file: rules + 4-phase workflow + routing
|
||||
├── references/
|
||||
│ ├── node-mapping.md ← Prompt Flow node → MAF mapping table + collapse patterns
|
||||
│ ├── workflow-context.md ← WorkflowContext types, LLM clients, ChatOptions, packages
|
||||
│ └── gotchas.md ← Common pitfalls, runtime errors, anti-patterns
|
||||
├── topics/
|
||||
│ ├── custom-tool-nodes.md ← Handling source.type: package nodes
|
||||
│ ├── multimodal.md ← Image/multimodal input handling
|
||||
│ └── evaluation-flows.md ← aggregation: true + EvalRunner pattern
|
||||
├── templates/
|
||||
│ └── eval_runner.py ← Reusable runner — copy verbatim into eval flow output
|
||||
└── examples/
|
||||
├── linear-chat.md ← Single LLM node + chat history
|
||||
├── multimodal-chat.md ← Image inputs (GPT-4V style)
|
||||
└── evaluation.md ← Per-row workflow + aggregation function + run_eval.py
|
||||
```
|
||||
@@ -0,0 +1,132 @@
|
||||
# Example: Evaluation Flow (Batch with Aggregation)
|
||||
|
||||
> Reference example. Read alongside [topics/evaluation-flows.md](../topics/evaluation-flows.md) when converting a flow with `aggregation: true`.
|
||||
|
||||
This converts an evaluation flow with a per-row `line_process` node and an `aggregation: true` node.
|
||||
|
||||
## Original `flow.dag.yaml`
|
||||
|
||||
```yaml
|
||||
nodes:
|
||||
- name: line_process
|
||||
type: python
|
||||
source:
|
||||
type: code
|
||||
path: line_process.py
|
||||
inputs:
|
||||
groundtruth: ${inputs.groundtruth}
|
||||
prediction: ${inputs.prediction}
|
||||
- name: aggregate
|
||||
type: python
|
||||
source:
|
||||
type: code
|
||||
path: aggregate.py
|
||||
inputs:
|
||||
processed_results: ${line_process.output}
|
||||
aggregation: true
|
||||
```
|
||||
|
||||
## `workflow.py` — Per-row workflow with factory function
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from typing_extensions import Never
|
||||
from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalInput:
|
||||
groundtruth: str
|
||||
prediction: str
|
||||
|
||||
|
||||
class LineProcessExecutor(Executor):
|
||||
@handler
|
||||
async def process(self, input: EvalInput, ctx: WorkflowContext[Never, str]) -> None:
|
||||
result = "Correct" if input.groundtruth.lower() == input.prediction.lower() else "Incorrect"
|
||||
await ctx.yield_output(result)
|
||||
|
||||
|
||||
def create_workflow():
|
||||
"""Create a fresh workflow instance.
|
||||
|
||||
MAF workflows do not support concurrent execution, so each batch row
|
||||
needs its own workflow instance.
|
||||
"""
|
||||
_line_process = LineProcessExecutor(id="line_process")
|
||||
return WorkflowBuilder(name="EvalBasicRow", start_executor=_line_process).build()
|
||||
```
|
||||
|
||||
## `aggregation.py` — Standalone aggregation function
|
||||
|
||||
```python
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def aggregate(processed_results: List[str]) -> Dict[str, int]:
|
||||
results_num = len(processed_results)
|
||||
correct_num = processed_results.count("Correct")
|
||||
return {
|
||||
"results_num": results_num,
|
||||
"correct_num": correct_num,
|
||||
}
|
||||
```
|
||||
|
||||
## `eval_runner.py`
|
||||
|
||||
Copy [templates/eval_runner.py](../templates/eval_runner.py) verbatim.
|
||||
|
||||
## `run_eval.py` — Entry point
|
||||
|
||||
```python
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from aggregation import aggregate
|
||||
from eval_runner import EvalRunner
|
||||
from workflow import EvalInput, create_workflow
|
||||
|
||||
DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
|
||||
|
||||
|
||||
def load_dataset(path: Path) -> list[EvalInput]:
|
||||
rows = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
obj = json.loads(line)
|
||||
rows.append(EvalInput(groundtruth=obj["groundtruth"], prediction=obj["prediction"]))
|
||||
return rows
|
||||
|
||||
|
||||
async def main(data_path: Path, concurrency: int):
|
||||
dataset = load_dataset(data_path)
|
||||
print(f"Loaded {len(dataset)} rows from {data_path}")
|
||||
|
||||
runner = EvalRunner(
|
||||
workflow_factory=create_workflow,
|
||||
aggregate_fn=aggregate,
|
||||
concurrency=concurrency,
|
||||
input_mapping={"values": "processed_results"},
|
||||
)
|
||||
result = await runner.run(dataset)
|
||||
|
||||
print(f"\n--- Metrics ---")
|
||||
for key, value in result.metrics.items():
|
||||
print(f" {key}: {value}")
|
||||
if result.errors:
|
||||
print(f"\n--- Errors ({len(result.errors)}) ---")
|
||||
for idx, err in result.errors:
|
||||
print(f" Row {idx}: {err}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
|
||||
parser.add_argument("--concurrency", type=int, default=5)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.data, args.concurrency))
|
||||
```
|
||||
@@ -0,0 +1,74 @@
|
||||
# Example: Linear Chat Flow
|
||||
|
||||
> Reference example. Read when you want a full template for a single-LLM-node flow with chat history.
|
||||
|
||||
This converts a Prompt Flow with one LLM node and chat history:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from dotenv import load_dotenv
|
||||
from typing_extensions import Never
|
||||
from agent_framework import Agent, Executor, WorkflowBuilder, WorkflowContext, handler
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@dataclass
|
||||
class ChatInput:
|
||||
question: str
|
||||
chat_history: list | None = None
|
||||
|
||||
class InputExecutor(Executor):
|
||||
@handler
|
||||
async def receive(self, chat_input: ChatInput, ctx: WorkflowContext[str]) -> None:
|
||||
parts = []
|
||||
if chat_input.chat_history:
|
||||
for turn in chat_input.chat_history:
|
||||
parts.append(f"User: {turn['inputs']['question']}")
|
||||
parts.append(f"Assistant: {turn['outputs']['answer']}")
|
||||
parts.append(chat_input.question)
|
||||
await ctx.send_message("\n".join(parts))
|
||||
|
||||
class ChatExecutor(Executor):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
client = OpenAIChatClient(
|
||||
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
model=os.environ["AZURE_OPENAI_DEPLOYMENT"],
|
||||
api_key=os.environ["AZURE_OPENAI_API_KEY"],
|
||||
)
|
||||
self._agent = Agent(
|
||||
client=client,
|
||||
name="ChatAgent",
|
||||
instructions="You are a helpful assistant.",
|
||||
)
|
||||
|
||||
@handler
|
||||
async def call_llm(self, question: str, ctx: WorkflowContext[Never, str]) -> None:
|
||||
response = await self._agent.run(question)
|
||||
await ctx.yield_output(response.text)
|
||||
|
||||
def create_workflow():
|
||||
"""Create a fresh workflow instance.
|
||||
|
||||
MAF workflows do not support concurrent execution, so each
|
||||
concurrent caller needs its own workflow instance.
|
||||
"""
|
||||
_input = InputExecutor(id="input")
|
||||
_chat = ChatExecutor(id="chat")
|
||||
return (
|
||||
WorkflowBuilder(name="BasicChatWorkflow", start_executor=_input)
|
||||
.add_edge(_input, _chat)
|
||||
.build()
|
||||
)
|
||||
|
||||
async def main():
|
||||
workflow = create_workflow()
|
||||
result = await workflow.run(ChatInput(question="What is ChatGPT?"))
|
||||
print(result.get_outputs()[0])
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
@@ -0,0 +1,113 @@
|
||||
# Example: Multimodal Chat Flow (Image + Text)
|
||||
|
||||
> Reference example. Read alongside [topics/multimodal.md](../topics/multimodal.md) when converting a flow with image inputs.
|
||||
|
||||
This converts a Prompt Flow with a `custom_llm` node that accepts image URLs (e.g., GPT-4V):
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from dotenv import load_dotenv
|
||||
from typing_extensions import Never
|
||||
from agent_framework import Agent, Content, Executor, Message, WorkflowBuilder, WorkflowContext, handler
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Matches Prompt Flow image key like "data:image/png;url"
|
||||
_IMAGE_KEY_RE = re.compile(r"^data:image/[^;]+;url$")
|
||||
# Matches Prompt Flow image string like "data:image/png;url: https://..."
|
||||
_IMAGE_STR_RE = re.compile(r"^data:image/[^;]+;url:\s*(.+)$")
|
||||
|
||||
|
||||
def _parse_question_parts(parts: list) -> list[Content | str]:
|
||||
"""Convert Prompt Flow multimodal question parts to Content objects.
|
||||
|
||||
Supports two formats:
|
||||
- dict: {"data:image/png;url": "https://example.com/img.png"}
|
||||
- string: "data:image/png;url: https://example.com/img.png"
|
||||
"""
|
||||
contents: list[Content | str] = []
|
||||
for part in parts:
|
||||
if isinstance(part, dict):
|
||||
for key, url in part.items():
|
||||
if _IMAGE_KEY_RE.match(key):
|
||||
contents.append(Content.from_uri(url, media_type="image/png"))
|
||||
elif isinstance(part, str):
|
||||
m = _IMAGE_STR_RE.match(part)
|
||||
if m:
|
||||
contents.append(Content.from_uri(m.group(1).strip(), media_type="image/png"))
|
||||
else:
|
||||
contents.append(part)
|
||||
else:
|
||||
contents.append(str(part))
|
||||
return contents
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatInput:
|
||||
question: list # e.g. [{"data:image/png;url": "<url>"}, "How many colors?"]
|
||||
chat_history: list = field(default_factory=list)
|
||||
|
||||
|
||||
class InputExecutor(Executor):
|
||||
@handler
|
||||
async def receive(self, chat_input: ChatInput, ctx: WorkflowContext[Message]) -> None:
|
||||
contents: list[Content | str] = []
|
||||
if chat_input.chat_history:
|
||||
for turn in chat_input.chat_history:
|
||||
contents.append(f"User: {turn['inputs']['question']}")
|
||||
contents.append(f"Assistant: {turn['outputs']['answer']}")
|
||||
contents.extend(_parse_question_parts(chat_input.question))
|
||||
await ctx.send_message(Message("user", contents))
|
||||
|
||||
|
||||
class ChatExecutor(Executor):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
client = OpenAIChatClient(
|
||||
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
model=os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4v"),
|
||||
api_key=os.environ["AZURE_OPENAI_API_KEY"],
|
||||
)
|
||||
self._agent = Agent(
|
||||
client=client,
|
||||
name="ChatImageAgent",
|
||||
instructions="You are a helpful assistant.",
|
||||
)
|
||||
|
||||
@handler
|
||||
async def call_llm(self, prompt: Message, ctx: WorkflowContext[Never, str]) -> None:
|
||||
response = await self._agent.run(prompt)
|
||||
await ctx.yield_output(response.text)
|
||||
|
||||
|
||||
def create_workflow():
|
||||
"""Create a fresh workflow instance.
|
||||
|
||||
MAF workflows do not support concurrent execution, so each
|
||||
concurrent caller needs its own workflow instance.
|
||||
"""
|
||||
_input = InputExecutor(id="input")
|
||||
_chat = ChatExecutor(id="chat")
|
||||
return (
|
||||
WorkflowBuilder(name="ChatWithImageWorkflow", start_executor=_input)
|
||||
.add_edge(_input, _chat)
|
||||
.build()
|
||||
)
|
||||
|
||||
async def main():
|
||||
workflow = create_workflow()
|
||||
result = await workflow.run(
|
||||
ChatInput(question=[
|
||||
"How many colors can you see?",
|
||||
{"data:image/png;url": "https://example.com/image.png"},
|
||||
])
|
||||
)
|
||||
print(result.get_outputs()[0])
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
@@ -0,0 +1,71 @@
|
||||
# Gotchas — Common Pitfalls When Converting Prompt Flow to MAF
|
||||
|
||||
> Read this every time you generate or debug MAF code. These are mistakes the agent makes repeatedly.
|
||||
|
||||
## High-frequency (almost always relevant)
|
||||
|
||||
### 1. `Agent.run()` returns `AgentResponse`, not `str`
|
||||
Always use `response.text` to get the string output, then pass that to `ctx.yield_output()`.
|
||||
|
||||
### 2. No `AzureOpenAIChatClient` class
|
||||
Use `OpenAIChatClient` with `azure_endpoint=...` for Azure routing.
|
||||
|
||||
### 3. `@handler` message type must match upstream
|
||||
If the upstream executor sends a `str` via `ctx.send_message(str)`, the downstream `@handler` parameter must be typed as `str`.
|
||||
|
||||
### 4. Never name a `@handler` method `execute`
|
||||
The base `Executor` class has an `execute()` method that the workflow engine calls with internal arguments (`message`, `source_executor_ids`, `state`, `runner_context`, `trace_contexts`, `source_span_ids`). If a subclass defines a `@handler` method also named `execute`, it shadows the base method, causing `TypeError: got an unexpected keyword argument 'trace_contexts'` at runtime. Use any other name (e.g., `run_code`, `process`, `handle`, `invoke`).
|
||||
|
||||
### 5. MAF workflows do not support concurrent `run()` calls
|
||||
Calling `workflow.run()` on an instance that is already running throws `RuntimeError: Workflow is already running. Concurrent executions are not allowed.` Always export a `create_workflow()` factory function and create a fresh instance per invocation. This applies to **all** workflows — not just evaluation flows — so callers can safely parallelize.
|
||||
|
||||
### 6. Preserve LLM parameters from the original flow
|
||||
If the Prompt Flow YAML sets `temperature`, `max_tokens`, etc. on an LLM node, these MUST be carried over to the MAF `Agent.run()` call via `OpenAIChatOptions`. Omitting them changes model behavior (e.g., higher temperature = less deterministic outputs, missing `max_tokens` = truncated/verbose responses).
|
||||
|
||||
### 7. LLM responses wrapped in markdown fences
|
||||
Modern LLMs often wrap JSON output in ` ```json ... ``` ` code fences even when not asked to. When parsing JSON from `Agent.run()` responses, always strip markdown fences before calling `json.loads()`:
|
||||
|
||||
```python
|
||||
text = response.text.strip()
|
||||
if text.startswith("```"):
|
||||
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
|
||||
result = json.loads(text)
|
||||
```
|
||||
|
||||
Without this, `json.loads()` raises `JSONDecodeError` and the fallback silently returns wrong results.
|
||||
|
||||
---
|
||||
|
||||
## Mid-frequency (situation-specific)
|
||||
|
||||
### 8. Chat history cannot be passed as `list[dict]` to `Agent.run()`
|
||||
Format it into a single prompt string instead.
|
||||
|
||||
### 9. Fan-in delivers `list[T]`
|
||||
The aggregator's `@handler` receives a list of all upstream messages.
|
||||
|
||||
### 10. Condition functions receive the message
|
||||
`condition=fn` where `fn(message) -> bool`.
|
||||
|
||||
### 11. Environment variables auto-read
|
||||
`OpenAIChatClient` can auto-read `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_CHAT_MODEL` from env, but explicit constructor args are clearer.
|
||||
|
||||
### 12. Internal package imports
|
||||
When a flow imports from sibling Python packages (e.g., `from my_utils.helpers import build_index`), copy the entire package directory into the MAF output folder. Do not rewrite utility code. The Executor files import directly from the local copy since they live in the same directory. Do not use `sys.path` hacks.
|
||||
|
||||
---
|
||||
|
||||
## Topic-specific (only when applicable)
|
||||
|
||||
### 13. Multimodal inputs require `Message`, not `str`
|
||||
When a flow has image inputs (e.g., GPT-4V), you must build a `Message("user", [Content.from_uri(...), "text"])` and pass it to `Agent.run()`. Joining image URLs into a plain string will NOT send the image to the model. See [topics/multimodal.md](../topics/multimodal.md).
|
||||
|
||||
### 14. Prompt Flow image format — handle both forms
|
||||
Prompt Flow image inputs come in two formats:
|
||||
- **Dict format** (from CLI): `{"data:image/png;url": "https://example.com/img.png"}` — extract the URL from the dict value
|
||||
- **String format** (from YAML defaults): `"data:image/png;url: https://example.com/img.png"` — parse the URL after `url: `
|
||||
|
||||
Both must be converted to `Content.from_uri(url, media_type="image/png")`.
|
||||
|
||||
### 15. Evaluation aggregation functions must return a dict
|
||||
The original PromptFlow aggregation nodes call `log_metric(key, value)` to report metrics. In MAF, replace these with a returned `dict` mapping metric names to values. Remove all `log_metric` imports and calls. See [topics/evaluation-flows.md](../topics/evaluation-flows.md).
|
||||
@@ -0,0 +1,78 @@
|
||||
# Node Mapping Reference
|
||||
|
||||
> Lookup table for converting Prompt Flow node types to MAF equivalents.
|
||||
> Read this when you need to map a specific Prompt Flow concept to MAF code.
|
||||
|
||||
## Core Mapping Table
|
||||
|
||||
| Prompt Flow Concept | MAF Equivalent |
|
||||
|---|---|
|
||||
| `flow.dag.yaml` (flow definition) | `WorkflowBuilder(name=..., start_executor=...).add_edge(...).build()` |
|
||||
| Any node | `Executor` subclass with a `@handler` method |
|
||||
| LLM node (`type: llm`) | `Agent(client=OpenAIChatClient(...), instructions=...)` inside an Executor |
|
||||
| Python node (`type: python`, `source.type: code`) | Plain Python logic inside an `Executor` `@handler` |
|
||||
| Custom-tool node (`type: python`, `source.type: package`) | **Call the tool's underlying Python function directly inside an `Executor` `@handler`. Do NOT remap to `OpenAIChatClient` / `Agent`.** See [topics/custom-tool-nodes.md](../topics/custom-tool-nodes.md). |
|
||||
| Prompt node (`.jinja2` template) | System prompt string passed to `Agent(instructions=...)`, or string formatting in `@handler` |
|
||||
| Conditional / If node (`activate_config`) | `.add_edge(source, target, condition=fn)` |
|
||||
| Parallel nodes (no shared deps) | `.add_fan_out_edges(source, [targetA, targetB])` |
|
||||
| Merge / aggregate node | `.add_fan_in_edges([sourceA, sourceB], target)` |
|
||||
| `aggregation: true` node (eval batch) | Standalone function + `EvalRunner` orchestrator. See [topics/evaluation-flows.md](../topics/evaluation-flows.md). |
|
||||
| Embed Text + Vector Lookup + LLM (RAG) | `AzureAISearchContextProvider` via `context_providers=[...]` on `Agent` |
|
||||
| Python tool node | Plain function passed to `Agent(tools=[fn1, fn2])` |
|
||||
| Flow inputs | Type annotation on start Executor's `@handler` parameter (use `@dataclass` for multiple inputs) |
|
||||
| Flow outputs (`is_chat_output`) | `await ctx.yield_output(value)` in the terminal Executor |
|
||||
| Connections (credentials) | Environment variables + `OpenAIChatClient(azure_endpoint=..., api_key=...)` for key auth, or `credential=DefaultAzureCredential()` for Microsoft Entra / managed identity. |
|
||||
| `chat_history` input | Format into prompt string in an InputExecutor before passing to Agent |
|
||||
| Variants | Separate Agent instances with different `instructions` strings |
|
||||
| Multimodal input (image URL) | `Content.from_uri(url, media_type="image/png")` inside a `Message` (see [topics/multimodal.md](../topics/multimodal.md)) |
|
||||
| Multimodal input (base64 image) | `Content.from_data(data=bytes, media_type="image/png")` inside a `Message` |
|
||||
| `custom_llm` node with images | Executor that builds a `Message("user", [Content.from_uri(...), text])` and passes it to `Agent.run()` |
|
||||
|
||||
## Node Collapsing Patterns
|
||||
|
||||
The default mapping is **1 PF node → 1 MAF Executor**, and the graph topology must be preserved (see SKILL.md Rule 13). The patterns below are the **only** merges allowed. Any merge MUST be annotated in the Phase 1 node-mapping table with the pattern name (e.g., "Collapsed via: Prompt + LLM"). If a combination is not on this list, keep the nodes separate as 1:1 Executors — do not invent new merges.
|
||||
|
||||
### Allowed merges (closed list)
|
||||
|
||||
- **Prompt template + LLM node** → Merge into one Executor: extract system prompt to `Agent(instructions=...)`, format user prompt as a string with variables, then call `Agent.run()`
|
||||
- Example: `hello_prompt` (`.jinja2`) + `llm` (LLM) → single `LLMExecutor` with both template and agent
|
||||
- Required: the prompt node must feed *only* the LLM node (no other downstream consumers).
|
||||
- **LLM + simple post-processing Python node** → Merge if post-processing is a few lines (e.g., extract substring, parse JSON, format output)
|
||||
- Required: post-processing is ≤ ~20 lines, has no external API calls, and the LLM node feeds *only* this post-processing node.
|
||||
- **Static-data Python node** → Inline as module-level constant (e.g., `prepare_examples()`, `math_example()`) if data is <50 lines and the node has no inputs (pure constant producer).
|
||||
|
||||
### When to keep separate
|
||||
|
||||
- Node would need concurrent execution (e.g., two branches from the same source run in parallel)
|
||||
- Post-processing is complex (>20 lines) or calls external APIs
|
||||
- Output is consumed by multiple downstream nodes (keep it separate for clarity and reuse)
|
||||
- Node has stateful side effects that should be isolated
|
||||
|
||||
### Example: Prompt + LLM collapse
|
||||
|
||||
```python
|
||||
# Instead of two Executors:
|
||||
class PromptExecutor(Executor):
|
||||
@handler
|
||||
async def receive(self, text: str, ctx: WorkflowContext[str]) -> None:
|
||||
prompt = f"Write a simple {text} program..."
|
||||
await ctx.send_message(prompt)
|
||||
|
||||
class LLMExecutor(Executor):
|
||||
@handler
|
||||
async def call_llm(self, prompt: str, ctx: WorkflowContext[Never, str]) -> None:
|
||||
response = await self._agent.run(prompt)
|
||||
await ctx.yield_output(response.text)
|
||||
|
||||
# Can merge into one:
|
||||
class PromptAndLLMExecutor(Executor):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._agent = Agent(client=..., instructions=...)
|
||||
|
||||
@handler
|
||||
async def generate(self, text: str, ctx: WorkflowContext[Never, str]) -> None:
|
||||
prompt = f"Write a simple {text} program..."
|
||||
response = await self._agent.run(prompt)
|
||||
await ctx.yield_output(response.text)
|
||||
```
|
||||
@@ -0,0 +1,85 @@
|
||||
# WorkflowContext, Client & ChatOptions Reference
|
||||
|
||||
> Quick lookup for type annotations, LLM client selection, and chat parameters.
|
||||
> Read this when writing Executor handlers or configuring LLM clients.
|
||||
|
||||
## WorkflowContext Type Parameters
|
||||
|
||||
| Annotation | Behaviour |
|
||||
|---|---|
|
||||
| `WorkflowContext` | Side effects only — no output sent |
|
||||
| `WorkflowContext[str]` | Sends a `str` downstream via `ctx.send_message()` |
|
||||
| `WorkflowContext[Never, str]` | Yields a `str` as the final workflow output via `ctx.yield_output()` |
|
||||
| `WorkflowContext[str, str]` | Both sends downstream AND yields a workflow output |
|
||||
|
||||
`Never` is imported from `typing_extensions`.
|
||||
|
||||
---
|
||||
|
||||
## LLM Client Selection
|
||||
|
||||
| Scenario | Client | Constructor |
|
||||
|----------|--------|-------------|
|
||||
| Azure OpenAI (API key) | `OpenAIChatClient` | `OpenAIChatClient(azure_endpoint=..., model=..., api_key=...)` |
|
||||
| Azure OpenAI (Entra ID) | `OpenAIChatClient` | `OpenAIChatClient(azure_endpoint=..., model=..., credential=DefaultAzureCredential())` |
|
||||
| OpenAI (direct) | `OpenAIChatClient` | `OpenAIChatClient(model=..., api_key=...)` |
|
||||
| Microsoft Foundry | `FoundryChatClient` | `FoundryChatClient(project_endpoint=..., model=..., credential=DefaultAzureCredential())` |
|
||||
|
||||
> `OpenAIChatClient` auto-routes to Azure when `azure_endpoint` is provided. There is no separate `AzureOpenAIChatClient` class.
|
||||
|
||||
---
|
||||
|
||||
## Chat Options (LLM Parameters)
|
||||
|
||||
Prompt Flow LLM nodes specify parameters like `temperature`, `max_tokens`, `top_p` in the YAML. In MAF, pass these via `OpenAIChatOptions` to `Agent.run()`:
|
||||
|
||||
```python
|
||||
from agent_framework.openai import OpenAIChatClient, OpenAIChatOptions
|
||||
|
||||
# In the @handler method:
|
||||
response = await self._agent.run(
|
||||
prompt,
|
||||
options=OpenAIChatOptions(temperature=0.2, max_tokens=128),
|
||||
)
|
||||
```
|
||||
|
||||
### Available Options
|
||||
|
||||
| Option | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `temperature` | `float` | Sampling temperature (0.0–2.0). Lower = more deterministic |
|
||||
| `max_tokens` | `int` | Maximum tokens in the response |
|
||||
| `top_p` | `float` | Nucleus sampling threshold |
|
||||
| `stop` | `str \| Sequence[str]` | Stop sequences |
|
||||
| `seed` | `int` | Deterministic sampling seed |
|
||||
| `frequency_penalty` | `float` | Penalize repeated tokens |
|
||||
| `presence_penalty` | `float` | Penalize tokens already present |
|
||||
| `response_format` | `type[BaseModel] \| dict` | Structured output schema |
|
||||
| `model` | `str` | Override the model for this call |
|
||||
| `tool_choice` | `str` | Tool selection mode (`auto`, `required`, `none`) |
|
||||
|
||||
### Mapping from Prompt Flow YAML
|
||||
|
||||
| Prompt Flow LLM node field | `OpenAIChatOptions` field |
|
||||
|---|---|
|
||||
| `temperature: '0.2'` | `temperature=0.2` |
|
||||
| `max_tokens: '128'` | `max_tokens=128` |
|
||||
| `top_p: '1.0'` | `top_p=1.0` |
|
||||
| `stop: ''` | (omit — empty means no stop sequence) |
|
||||
| `frequency_penalty: '0'` | (omit — 0 is the default) |
|
||||
| `presence_penalty: '0'` | (omit — 0 is the default) |
|
||||
|
||||
> Prompt Flow YAML stores these as strings (e.g., `'0.2'`). Convert to the appropriate numeric type in `OpenAIChatOptions`.
|
||||
|
||||
---
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Version | Purpose |
|
||||
|---------|---------|---------|
|
||||
| `agent-framework` | >=1.0.1 (GA) | Core: `Executor`, `WorkflowBuilder`, `WorkflowContext`, `Agent`, `@handler` |
|
||||
| `agent-framework-openai` | >=1.0.1 (GA) | `OpenAIChatClient`, `OpenAIChatOptions` — works for both OpenAI and Azure OpenAI |
|
||||
| `agent-framework-foundry` | >=1.0.1 (GA) | `FoundryChatClient` — for Microsoft Foundry endpoints |
|
||||
| `agent-framework-orchestrations` | preview | `HandoffBuilder` — for multi-agent handoffs |
|
||||
| `agent-framework-azure-ai-search` | preview | `AzureAISearchContextProvider` — for RAG pipelines |
|
||||
| `python-dotenv` | any | Load `.env` for credentials |
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Reusable EvalRunner for MAF-based evaluation flows.
|
||||
|
||||
Copy this file verbatim into the output folder of a converted evaluation flow.
|
||||
|
||||
Mirrors PromptFlow's two-phase execution model:
|
||||
Phase 1 — run each row through the workflow concurrently
|
||||
Phase 2 — pass all collected outputs to the aggregation function
|
||||
|
||||
MAF workflows do not support concurrent execution on a single instance,
|
||||
so workflow_factory creates a fresh workflow for each concurrent row.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalResult:
|
||||
"""Result of a batch evaluation run."""
|
||||
per_row_outputs: List[Any]
|
||||
metrics: Dict[str, Any]
|
||||
errors: List[tuple] = field(default_factory=list)
|
||||
|
||||
|
||||
class EvalRunner:
|
||||
"""Runs a MAF workflow per row, collects outputs, then calls an aggregation function."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
workflow_factory: Callable[[], Any],
|
||||
aggregate_fn: Callable[..., dict],
|
||||
concurrency: int = 5,
|
||||
input_mapping: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._workflow_factory = workflow_factory
|
||||
self._aggregate_fn = aggregate_fn
|
||||
self._concurrency = concurrency
|
||||
self._input_mapping = input_mapping
|
||||
|
||||
async def run(self, dataset: List[Any]) -> EvalResult:
|
||||
semaphore = asyncio.Semaphore(self._concurrency)
|
||||
per_row_outputs: List[Any] = [None] * len(dataset)
|
||||
errors: List[tuple] = []
|
||||
|
||||
async def _run_row(index: int, row: Any) -> None:
|
||||
async with semaphore:
|
||||
wf = self._workflow_factory()
|
||||
result = await wf.run(row)
|
||||
per_row_outputs[index] = result.get_outputs()[0]
|
||||
|
||||
tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
succeeded_outputs: List[Any] = []
|
||||
for i, r in enumerate(results):
|
||||
if isinstance(r, Exception):
|
||||
errors.append((i, r))
|
||||
else:
|
||||
succeeded_outputs.append(per_row_outputs[i])
|
||||
|
||||
aggregation_inputs = self._transpose(succeeded_outputs)
|
||||
if self._input_mapping:
|
||||
aggregation_inputs = {
|
||||
self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
|
||||
}
|
||||
|
||||
metrics = self._aggregate_fn(**aggregation_inputs)
|
||||
return EvalResult(
|
||||
per_row_outputs=succeeded_outputs,
|
||||
metrics=metrics,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _transpose(outputs: List[Any]) -> Dict[str, Any]:
|
||||
if not outputs:
|
||||
return {"values": []}
|
||||
if not isinstance(outputs[0], dict):
|
||||
return {"values": outputs}
|
||||
keys = outputs[0].keys()
|
||||
return {k: [o[k] for o in outputs] for k in keys}
|
||||
@@ -0,0 +1,97 @@
|
||||
# Custom-Tool Nodes (`source.type: package`)
|
||||
|
||||
> **Read this when** the source `flow.dag.yaml` contains a node with `source.type: package`.
|
||||
|
||||
A node whose `source.type` is `package` (rather than `code`) is a **user-defined PromptFlow tool**, not a stock LLM/Python node. The YAML looks like:
|
||||
|
||||
```yaml
|
||||
- name: my_node
|
||||
type: python
|
||||
source:
|
||||
type: package
|
||||
tool: my_pkg.my_module.MyToolClass.my_function # ← package-qualified tool path
|
||||
inputs:
|
||||
connection: my_connection
|
||||
prompt: ${upstream.output}
|
||||
model: my-internal-model-name
|
||||
# ...other tool-specific params declared by the tool's function signature
|
||||
```
|
||||
|
||||
## Rule: keep the tool, replace only the runtime
|
||||
|
||||
Custom tools wrap functionality that MAF **does not provide**:
|
||||
- Internal/proprietary LLM gateways (custom endpoints, auth, quota, batch routing)
|
||||
- Domain-specific clients (search backends, vector stores, internal APIs)
|
||||
- Org-specific connection types (`CustomConnection`, vendor SDK clients)
|
||||
- Special headers, model-name conventions, segmentation, retry policies
|
||||
|
||||
**Re-pointing these calls to `OpenAIChatClient` / `Agent` silently changes the wire protocol, endpoint, model, auth, and quota system — and almost always breaks the deployment.** Even if the tool happens to wrap an OpenAI-compatible API, tool-specific parameters and behaviors (custom routing flags, quota tags, headers, model name conventions) will not survive the remap.
|
||||
|
||||
## Conversion steps
|
||||
|
||||
1. **Locate the tool implementation.** The `tool:` field is a Python import path. Open the source file and identify:
|
||||
- The class / function name (the last segment after the final `.`)
|
||||
- The class above it (second-to-last segment) — usually a `ToolProvider` subclass with a `@tool`-decorated method
|
||||
- Whether the `@tool` method delegates to a plain underlying class (very common pattern) — prefer calling that underlying class directly to drop the PromptFlow runtime dependency
|
||||
2. **Read the function signature.** Note every parameter the YAML node passes, plus any defaults the tool applies internally (endpoints, headers, auth scopes, etc.).
|
||||
3. **Inspect the connection object.** If the tool takes a `connection: CustomConnection`, find out where it reads its credentials. Many custom tools resolve credentials from environment variables (managed identity, AAD, client secret) and ignore the `connection` arg entirely — in which case pass `None` or omit it.
|
||||
4. **Call the tool from inside an `Executor` `@handler`.** Instantiate the tool class once in `__init__` (so any credential/token caching is reused), then invoke it inside the handler.
|
||||
5. **Wrap synchronous I/O in `asyncio.to_thread`.** Most custom tools are synchronous (they call `requests.post`, vendor SDKs, etc.). Calling them directly from an `async def` handler blocks the event loop — wrap them so other executors can run concurrently:
|
||||
```python
|
||||
result = await asyncio.to_thread(self._tool, **tool_kwargs)
|
||||
```
|
||||
6. **Pass parameters verbatim from the YAML.** Copy every input from the node's `inputs:` block — including any tool-specific ones the original author added. Do not drop them; the tool's behavior may depend on them.
|
||||
7. **Add the tool's package to `requirements.txt`.** If the tool lives in an in-repo package, document the install path (e.g., `pip install -e ../path/to/package`).
|
||||
8. **Document credential env vars in `.env.example`.** Mirror whatever auth scheme the tool uses (managed identity, service principal, custom token endpoint).
|
||||
|
||||
## Example
|
||||
|
||||
Original PromptFlow node:
|
||||
|
||||
```yaml
|
||||
- name: my_llm_call
|
||||
type: python
|
||||
source:
|
||||
type: package
|
||||
tool: my_pkg.tools.gateway.GatewayCompletion.completion
|
||||
inputs:
|
||||
connection: my_conn
|
||||
prompt: ${prompt_node.output}
|
||||
model: internal-model-v2
|
||||
max_tokens: 500
|
||||
temperature: 0
|
||||
# ...any other tool-specific params declared by the tool's signature
|
||||
```
|
||||
|
||||
MAF Executor:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from agent_framework import Executor, WorkflowContext, handler
|
||||
from my_pkg.tools.gateway import GatewayCompletion # the underlying class
|
||||
|
||||
class MyLLMExecutor(Executor):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
# Instantiate once so token/credential caching is reused across calls.
|
||||
self._llm = GatewayCompletion()
|
||||
|
||||
@handler
|
||||
async def call(self, prompt: str, ctx: WorkflowContext[str]) -> None:
|
||||
text = await asyncio.to_thread(
|
||||
self._llm,
|
||||
prompt=prompt,
|
||||
model="internal-model-v2",
|
||||
max_tokens=500,
|
||||
temperature=0,
|
||||
# ...forward every other input from the YAML node verbatim
|
||||
)
|
||||
await ctx.send_message(text)
|
||||
```
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- ❌ Replacing the custom tool with `OpenAIChatClient` because "it's an LLM call too" — endpoint, auth, model names, and gateway-specific routing will all be wrong.
|
||||
- ❌ Re-implementing the tool's HTTP request inline — duplicates auth/header/retry logic that the original tool already handles correctly.
|
||||
- ❌ Importing the tool's `@tool`-decorated wrapper just to call it — pulls in PromptFlow as a runtime dependency. Prefer the underlying class.
|
||||
- ❌ Calling the synchronous tool directly from an `async` handler — blocks the event loop and kills fan-out concurrency. Always wrap in `asyncio.to_thread`.
|
||||
@@ -0,0 +1,73 @@
|
||||
# Evaluation Flows (`aggregation: true`)
|
||||
|
||||
> **Read this when** any node in `flow.dag.yaml` has `aggregation: true`.
|
||||
|
||||
An evaluation flow has two phases:
|
||||
- **Per-row phase** — runs the non-aggregation nodes once per dataset row (concurrently)
|
||||
- **Aggregation phase** — collects all per-row outputs and computes batch metrics
|
||||
|
||||
In MAF, this maps to:
|
||||
- A per-row `WorkflowBuilder` workflow (no aggregation node)
|
||||
- A standalone Python aggregation function
|
||||
- An `EvalRunner` orchestrator that runs the workflow per row and feeds outputs to the aggregator
|
||||
|
||||
## File layout
|
||||
|
||||
The generated `<original>-maf/` folder for an evaluation flow must contain:
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `workflow.py` | Per-row MAF workflow + `create_workflow()` factory |
|
||||
| `aggregation.py` | Standalone aggregation function(s) extracted from the original `aggregation: true` node(s) |
|
||||
| `eval_runner.py` | Reusable `EvalRunner` class — copy [templates/eval_runner.py](../templates/eval_runner.py) verbatim |
|
||||
| `run_eval.py` | Entry point: load dataset, instantiate `EvalRunner`, print metrics |
|
||||
| `requirements.txt`, `.env.example` | As usual |
|
||||
|
||||
## Phase 1 — Audit additions
|
||||
|
||||
When auditing the source flow, also identify:
|
||||
|
||||
- **Per-row nodes** — all nodes WITHOUT `aggregation: true`
|
||||
- **Aggregation nodes** — all nodes WITH `aggregation: true`
|
||||
- **Aggregation inputs** — which per-row node outputs feed into the aggregation node (e.g., `${grade.output}` → `grades: List[str]`)
|
||||
|
||||
## Generating the per-row `workflow.py`
|
||||
|
||||
Build a `WorkflowBuilder` containing **only** the non-aggregation nodes. Export a `create_workflow()` factory function (not a module-level singleton) so `EvalRunner` can create a fresh instance per concurrent row.
|
||||
|
||||
## Generating `aggregation.py`
|
||||
|
||||
Extract each aggregation node's Python function as a standalone function:
|
||||
|
||||
- Remove the `@tool` decorator
|
||||
- Remove `from promptflow.core import log_metric` and all `log_metric()` calls
|
||||
- Instead of calling `log_metric(key, value)`, include the metric in the returned `dict`
|
||||
- Keep the function signature (parameter names and types) identical to the original
|
||||
- The function must return a `dict` mapping metric names to values
|
||||
|
||||
## Generating `run_eval.py`
|
||||
|
||||
Configure the `EvalRunner`:
|
||||
|
||||
- `workflow_factory` → the `create_workflow` function from `workflow.py`
|
||||
- `aggregate_fn` → the aggregation function from `aggregation.py`
|
||||
- `input_mapping` → maps transposed key names to aggregation function parameter names
|
||||
|
||||
### Input mapping rules
|
||||
|
||||
`EvalRunner._transpose()` converts per-row outputs into keyword args for the aggregation function:
|
||||
|
||||
| Per-row output type | `_transpose()` produces | `input_mapping` needed? |
|
||||
|---|---|---|
|
||||
| Plain value (`str`, `int`, `float`) | `{"values": [v1, v2, ...]}` | Yes — map `"values"` → aggregation param name (e.g., `{"values": "processed_results"}`) |
|
||||
| Dict (e.g., `{"coherence": 4.2, "fluency": 2.5}`) | `{"coherence": [4.2, ...], "fluency": [2.5, ...]}` | Only if dict keys differ from aggregation param names |
|
||||
|
||||
For multi-output flows (e.g., `eval-summarization` with 4 scores per row), the per-row workflow should yield a dict whose keys match the aggregation function's parameter names. Then no `input_mapping` is needed.
|
||||
|
||||
## EvalRunner template
|
||||
|
||||
Copy [templates/eval_runner.py](../templates/eval_runner.py) verbatim into the output folder. It is identical across all evaluation flows.
|
||||
|
||||
## Complete example
|
||||
|
||||
See [examples/evaluation.md](../examples/evaluation.md) for a full converted evaluation flow.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Multimodal Inputs (Images + Text)
|
||||
|
||||
> **Read this when** the source flow has image inputs, uses a `custom_llm` node with images, or targets a vision model (e.g., GPT-4V, GPT-4o vision).
|
||||
|
||||
## Key rule
|
||||
|
||||
When a flow has image inputs, you must build a `Message("user", [Content.from_uri(...), "text"])` and pass it to `Agent.run()`. **Joining image URLs into a plain string will NOT send the image to the model.**
|
||||
|
||||
The downstream Executor's `@handler` must accept `Message` (not `str`) and `WorkflowContext` must use `Message` as the send type.
|
||||
|
||||
## Multimodal Content Reference
|
||||
|
||||
`Agent.run()` accepts `AgentRunInputs = str | Content | Message | Sequence[str | Content | Message]`.
|
||||
|
||||
For multimodal inputs (images + text), use `Message` with mixed content:
|
||||
|
||||
| Input Type | How to Create |
|
||||
|---|---|
|
||||
| Image from URL | `Content.from_uri("https://example.com/img.png", media_type="image/png")` |
|
||||
| Image from bytes | `Content.from_data(data=image_bytes, media_type="image/png")` |
|
||||
| Image from base64 data URI | `Content.from_uri("data:image/png;base64,iVBOR...")` |
|
||||
| Mixed image + text | `Message("user", [Content.from_uri(url, media_type="image/png"), "Describe this"])` |
|
||||
|
||||
## Prompt Flow Image Input Formats
|
||||
|
||||
Prompt Flow image inputs come in two formats — **both must be handled**:
|
||||
|
||||
- **Dict format** (from CLI): `{"data:image/png;url": "https://example.com/img.png"}` — extract the URL from the dict value
|
||||
- **String format** (from YAML defaults): `"data:image/png;url: https://example.com/img.png"` — parse the URL after `url: `
|
||||
|
||||
For dicts, match the key against `data:image/...;url` and extract the value as the URL.
|
||||
For strings, parse the URL after `url: ` using a regex.
|
||||
|
||||
For base64-encoded images, use `Content.from_data(data=image_bytes, media_type="image/...")`.
|
||||
|
||||
## Conversion checklist for multimodal nodes
|
||||
|
||||
1. Detect any input field whose value is a dict with key `data:image/*;url` or a string starting with `data:image/`.
|
||||
2. Build a parser that handles both formats and yields a list of `Content | str` items.
|
||||
3. In the InputExecutor, emit a `Message("user", contents)` instead of a plain string.
|
||||
4. The downstream LLM Executor's `@handler` parameter must be `Message`, and the upstream `WorkflowContext[Message]` must declare `Message` as the send type.
|
||||
5. Pass the `Message` directly to `Agent.run()` — do not stringify it.
|
||||
|
||||
See [examples/multimodal-chat.md](../examples/multimodal-chat.md) for a complete runnable example.
|
||||
@@ -0,0 +1,200 @@
|
||||
# Node Variants (`node_variants` / `use_variants`)
|
||||
|
||||
> **Read this when** the source `flow.dag.yaml` contains a `node_variants:` block, or any node has `use_variants: true`.
|
||||
|
||||
## What `node_variants` means in Prompt Flow
|
||||
|
||||
`node_variants` lets a single node slot have **multiple alternative configurations** (different prompts, model parameters, connections, even different node bodies). At run time PF picks **exactly one** variant per node — the one named by `--variant '${node.variant_x}'`, or the node's `default_variant_id` if nothing is specified. Other variants do **not** execute.
|
||||
|
||||
```yaml
|
||||
- name: summarize_text_content
|
||||
use_variants: true # ← node body is in node_variants below
|
||||
|
||||
node_variants:
|
||||
summarize_text_content:
|
||||
default_variant_id: variant_0
|
||||
variants:
|
||||
variant_0: { node: { type: llm, ... temperature: 0.2, ... } }
|
||||
variant_1: { node: { type: llm, ... temperature: 0.3, ... } }
|
||||
```
|
||||
|
||||
So conceptually a variant is a **swap-in node body**, and only one is "live" per run.
|
||||
|
||||
## MAF has no `variants` primitive
|
||||
|
||||
Microsoft Agent Framework has no first-class concept of "node variants". You implement the same behavior with plain Python: build the workflow with whichever configuration the caller asked for. Two patterns cover every real flow.
|
||||
|
||||
---
|
||||
|
||||
## Pattern A — Single Executor + `variant` parameter (default)
|
||||
|
||||
Use when the variants differ **only** in:
|
||||
- Prompt text (different `.jinja2` content)
|
||||
- LLM parameters (`temperature`, `max_tokens`, `top_p`, ...)
|
||||
- Model / deployment name
|
||||
- Connection (endpoint / api key)
|
||||
|
||||
This covers the vast majority of real Prompt Flow variants. The Executor stays one class; the variant just selects from a config dict.
|
||||
|
||||
```python
|
||||
VARIANT_INSTRUCTIONS = {
|
||||
"variant_0": "You are an assistant... return only the final answer.",
|
||||
"variant_1": "You are an assistant... think step by step. Return JSON.",
|
||||
"variant_2": "You are an assistant... here are 5 examples: ...",
|
||||
}
|
||||
|
||||
# Optional: per-variant LLM parameters
|
||||
VARIANT_OPTIONS = {
|
||||
"variant_0": {"temperature": 0.2, "max_tokens": 128},
|
||||
"variant_1": {"temperature": 0.3, "max_tokens": 256},
|
||||
"variant_2": {"temperature": 0.3, "max_tokens": 256},
|
||||
}
|
||||
|
||||
|
||||
class ChatExecutor(Executor):
|
||||
def __init__(self, variant: str = "variant_0", **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._opts = VARIANT_OPTIONS[variant]
|
||||
self._agent = Agent(
|
||||
client=OpenAIChatClient(
|
||||
azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
model=os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4"),
|
||||
api_key=os.environ["AZURE_OPENAI_API_KEY"],
|
||||
),
|
||||
instructions=VARIANT_INSTRUCTIONS[variant],
|
||||
)
|
||||
|
||||
@handler
|
||||
async def call_llm(self, prompt: str, ctx: WorkflowContext[str]) -> None:
|
||||
response = await self._agent.run(prompt, **self._opts)
|
||||
await ctx.send_message(response.text)
|
||||
|
||||
|
||||
def create_workflow(variant: str = "variant_0"):
|
||||
_input = InputExecutor(id="input")
|
||||
_chat = ChatExecutor(id="chat", variant=variant)
|
||||
_extract = ExtractResultExecutor(id="extract_result")
|
||||
return (
|
||||
WorkflowBuilder(name="ChatMathVariantWorkflow", start_executor=_input)
|
||||
.add_edge(_input, _chat)
|
||||
.add_edge(_chat, _extract)
|
||||
.build()
|
||||
)
|
||||
```
|
||||
|
||||
Mapping back to PF concepts:
|
||||
|
||||
| Prompt Flow | MAF (Pattern A) |
|
||||
|---|---|
|
||||
| `node_variants:` block | `VARIANT_INSTRUCTIONS` (+ optional `VARIANT_OPTIONS`) dict |
|
||||
| `default_variant_id: variant_0` | `def create_workflow(variant: str = "variant_0")` default |
|
||||
| `use_variants: true` node | Single Executor that takes a `variant` arg |
|
||||
| `pf run --variant '${node.variant_1}'` | `create_workflow(variant="variant_1")` (or env var) |
|
||||
|
||||
A complete reference is [chat-math-variant-maf](../../../../examples/flows/chat/chat-math-variant-maf/workflow.py).
|
||||
|
||||
---
|
||||
|
||||
## Pattern B — One Executor per variant
|
||||
|
||||
Use **only** when variants differ in something more than prompt/parameters:
|
||||
|
||||
1. **Different node `type`** — e.g. `variant_0` is `type: llm`, `variant_1` is `type: python`. The clients, dependencies, and error-handling diverge enough that a single class becomes a tangle of `if variant == ...`.
|
||||
2. **Different tools / `context_providers`** — e.g. `variant_1` adds an Azure AI Search retriever or extra Python tools to the Agent. Tool sets are decided at construction, not per-call.
|
||||
3. **Different input/output schema** — `@handler` signatures are strongly typed, and Pattern A can't change them per variant.
|
||||
4. **Different upstream/downstream connections** — e.g. `variant_1` needs an extra preprocessing step. The graph itself changes, so the `WorkflowBuilder` must branch (whether or not the Executor splits is secondary).
|
||||
5. **A/B comparison in a single run** — you want to fan out to all variants at once and compare. PF can't do this natively, but MAF can with `add_fan_out_edges` / `add_fan_in_edges`.
|
||||
|
||||
Skeleton:
|
||||
|
||||
```python
|
||||
class Variant0Executor(Executor): ...
|
||||
class Variant1Executor(Executor): ... # different tools, schema, or type
|
||||
|
||||
EXECUTORS = {"variant_0": Variant0Executor, "variant_1": Variant1Executor}
|
||||
|
||||
|
||||
def create_workflow(variant: str = "variant_0"):
|
||||
_input = InputExecutor(id="input")
|
||||
_chat = EXECUTORS[variant](id="chat")
|
||||
_post = PostExecutor(id="post")
|
||||
|
||||
builder = WorkflowBuilder(name="...", start_executor=_input).add_edge(_input, _chat)
|
||||
|
||||
# Variant-specific upstream/downstream wiring (case 4 above)
|
||||
if variant == "variant_1":
|
||||
_retrieve = RetrieveExecutor(id="retrieve")
|
||||
builder = builder.add_edge(_input, _retrieve).add_edge(_retrieve, _chat)
|
||||
|
||||
return builder.add_edge(_chat, _post).build()
|
||||
```
|
||||
|
||||
A/B fan-out variant of Pattern B (run all variants for evaluation):
|
||||
|
||||
```python
|
||||
def create_ab_workflow():
|
||||
_input = InputExecutor(id="input")
|
||||
branches = [Variant0Executor(id="v0"), Variant1Executor(id="v1"), Variant2Executor(id="v2")]
|
||||
_compare = CompareExecutor(id="compare")
|
||||
return (
|
||||
WorkflowBuilder(name="ABCompare", start_executor=_input)
|
||||
.add_fan_out_edges(_input, branches)
|
||||
.add_fan_in_edges(branches, _compare)
|
||||
.build()
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Decision tree
|
||||
|
||||
```
|
||||
Variants differ in...
|
||||
├── prompt text only → Pattern A
|
||||
├── prompt + LLM params (temperature, etc.) → Pattern A (+ VARIANT_OPTIONS dict)
|
||||
├── connection / model / deployment → Pattern A (+ per-variant client kwargs)
|
||||
├── node type (llm vs python vs custom tool) → Pattern B
|
||||
├── tool set / context_providers → Pattern B
|
||||
├── input/output schema (handler signature) → Pattern B
|
||||
├── upstream/downstream edges → builder branches (Pattern A or B)
|
||||
└── must compare all variants in one run → Pattern B + fan_out / fan_in
|
||||
```
|
||||
|
||||
When in doubt, start with **Pattern A**. It is closer to the PF mental model ("one node slot, swap-in config") and produces the smallest diff. Promote to Pattern B only when one of the conditions above forces it.
|
||||
|
||||
---
|
||||
|
||||
## Multiple nodes with variants
|
||||
|
||||
PF allows `node_variants` on several nodes; a single `pf run` picks one variant per node (others fall back to their `default_variant_id`). Keep the same shape in MAF — accept a dict so callers can mix and match without an exponential class explosion:
|
||||
|
||||
```python
|
||||
def create_workflow(variants: dict[str, str] | None = None):
|
||||
variants = variants or {}
|
||||
_summarize = SummarizeExecutor(
|
||||
id="summarize_text_content",
|
||||
variant=variants.get("summarize_text_content", "variant_0"),
|
||||
)
|
||||
_classify = ClassifyExecutor(
|
||||
id="classify_with_llm",
|
||||
variant=variants.get("classify_with_llm", "variant_0"),
|
||||
)
|
||||
...
|
||||
```
|
||||
|
||||
Caller:
|
||||
|
||||
```python
|
||||
wf = create_workflow(variants={"summarize_text_content": "variant_1"})
|
||||
```
|
||||
|
||||
This is the direct equivalent of `pf run --variant '${summarize_text_content.variant_1}'`, and it scales: two nodes with 3 and 2 variants respectively stay 2 Executor classes (Pattern A) instead of becoming 6 (Pattern B).
|
||||
|
||||
---
|
||||
|
||||
## Output rules (in addition to the Core Rules in `SKILL.md`)
|
||||
|
||||
- Always preserve every variant's prompt **verbatim** (Core Rule 2). Each variant's `.jinja2` becomes its own constant (`VARIANT_0_INSTRUCTIONS`, etc.) — do not merge or deduplicate prompt text even if variants share large prefixes.
|
||||
- The default value of the `variant` parameter must equal `default_variant_id` from the YAML.
|
||||
- If only one variant exists in `node_variants` (sometimes used as a poor-man's prompt holder), collapse it to a plain Executor with no `variant` parameter — there is nothing to switch.
|
||||
- Document the variant switch in the generated `.env.example` (e.g., `CHAT_VARIANT=variant_0`) and the `test_<name>.py` sample.
|
||||
@@ -0,0 +1,89 @@
|
||||
name: Build Doc CI
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'README.md'
|
||||
- 'docs/**'
|
||||
- 'examples/**.ipynb'
|
||||
- 'scripts/docs/**'
|
||||
- '.github/workflows/build_doc_ci.yml'
|
||||
- 'src/promptflow-tracing/promptflow/**'
|
||||
- 'src/promptflow-core/promptflow/**'
|
||||
- 'src/promptflow-devkit/promptflow/**'
|
||||
- 'src/promptflow-azure/promptflow/**'
|
||||
- 'src/promptflow-rag/promptflow/**'
|
||||
- 'src/promptflow-evals/promptflow/**'
|
||||
|
||||
env:
|
||||
packageSetupType: promptflow_with_extra
|
||||
testWorkingDirectory: ${{ github.workspace }}/src/promptflow
|
||||
|
||||
jobs:
|
||||
build_doc_job:
|
||||
runs-on: windows-latest
|
||||
name: Build Doc
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Python Setup
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
|
||||
- name: Install packages
|
||||
shell: pwsh
|
||||
# Note: Use -e to avoid duplicate object warning when build apidoc.
|
||||
run: |
|
||||
pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-core
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-devkit
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-azure
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-rag
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-evals
|
||||
pip freeze
|
||||
|
||||
- name: Build doc with reference doc
|
||||
shell: powershell
|
||||
working-directory: scripts/docs/
|
||||
run: |-
|
||||
pip install langchain tenacity<8.4.0
|
||||
./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true
|
||||
|
||||
# Note: We have this job separately because some error may missing when build link check exists.
|
||||
link_check_job:
|
||||
runs-on: windows-latest
|
||||
name: Build Link Check
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Python Setup
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
|
||||
- name: Install packages
|
||||
shell: pwsh
|
||||
# Note: Use -e to avoid duplicate object warning when build apidoc.
|
||||
run: |
|
||||
pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-core
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-devkit
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-azure
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-rag
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-evals
|
||||
pip freeze
|
||||
|
||||
- name: Build LinkCheck
|
||||
shell: powershell
|
||||
working-directory: scripts/docs/
|
||||
run: |-
|
||||
pip install langchain tenacity<8.4.0
|
||||
./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true -BuildLinkCheck
|
||||
@@ -0,0 +1,238 @@
|
||||
name: Build and Package MSI & Portable Installer
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
uploadAsLatest:
|
||||
type: string
|
||||
default: "False"
|
||||
required: false
|
||||
description: 'Also upload the msi installer to storage account as latest'
|
||||
|
||||
version:
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
description: 'Version of promptflow to install (optional). Will build locally if not specified.'
|
||||
|
||||
set_msi_private_version:
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
description: 'Set the version of the private msi installer'
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
packageSetupType: promptflow_with_extra
|
||||
testWorkingDirectory: src/promptflow
|
||||
AZURE_ACCOUNT_NAME: promptflowartifact
|
||||
AZURE_MSI_CONTAINER_NAME: msi-installer
|
||||
AZURE_PORTABLE_CONTAINER_NAME: portable-installer
|
||||
|
||||
jobs:
|
||||
build_msi_installer:
|
||||
runs-on: windows-latest
|
||||
name: Build Windows MSI
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Check input parameters
|
||||
run: |
|
||||
echo "uploadAsLatest: ${{ inputs.uploadAsLatest }}"
|
||||
echo "version: ${{ inputs.version }}"
|
||||
echo "set_msi_private_version: ${{ inputs.set_msi_private_version }}"
|
||||
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Add msbuild to PATH
|
||||
uses: microsoft/setup-msbuild@v1.1
|
||||
|
||||
- name: Install WIX Toolset
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}/scripts/installer/windows
|
||||
run: |
|
||||
Invoke-WebRequest -Uri 'https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip' -OutFile 'wix-archive.zip'
|
||||
Expand-Archive -Path 'wix-archive.zip' -DestinationPath 'wix'
|
||||
Remove-Item -Path 'wix-archive.zip'
|
||||
|
||||
- name: Python Setup
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
|
||||
- name: Install stable promptflow
|
||||
if: ${{ github.event.inputs.version != null && github.event.inputs.version != '' }}
|
||||
run: |
|
||||
pip install "promptflow[azure,executable,azureml-serving,executor-service]==$env:INPUT_VERSION" promptflow-tools
|
||||
echo "There's no promptflow-evals in pypi, we need to install it from source for now"
|
||||
pip install ${{ github.workspace }}/src/promptflow-evals
|
||||
env:
|
||||
INPUT_VERSION: ${{ github.event.inputs.version }}
|
||||
shell: pwsh
|
||||
|
||||
- name: Get promptflow version
|
||||
id: get-version
|
||||
# Convert string to int since the version tuple used in "version_info" can't start with 0.
|
||||
run: |
|
||||
if ($env:INPUT_VERSION) {
|
||||
$version=$env:INPUT_VERSION
|
||||
$run_version=$(python -c "import promptflow; print(promptflow.__version__)")
|
||||
if ($version -ne $run_version) {
|
||||
throw "Version input does not match the version in promptflow package. Version input: $version, version in promptflow package: $run_version"
|
||||
}
|
||||
}
|
||||
elseif ($env:MSI_PRIVATE_VERSION) {
|
||||
$version=$env:MSI_PRIVATE_VERSION
|
||||
}
|
||||
else {
|
||||
$prefix = 0
|
||||
$year = [int](Get-Date -Format "yy")
|
||||
$monthday = [int](Get-Date -Format "MMdd")
|
||||
$hourminutesecond = [int](Get-Date -Format "HHmmss")
|
||||
$version="$prefix.$year.$monthday.$hourminutesecond"
|
||||
}
|
||||
echo "::set-output name=version::$version"
|
||||
env:
|
||||
INPUT_VERSION: ${{ github.event.inputs.version }}
|
||||
MSI_PRIVATE_VERSION: ${{ github.event.inputs.set_msi_private_version }}
|
||||
shell: pwsh
|
||||
|
||||
- name: Update promptflow package version when set msi private version
|
||||
if: ${{ github.event.inputs.set_msi_private_version != null && github.event.inputs.set_msi_private_version != '' }}
|
||||
run: |
|
||||
$override_version = 'VERSION = "{0}"' -f $env:MSI_PRIVATE_VERSION
|
||||
Write-Host "'$override_version' as version"
|
||||
$override_version | Out-File -FilePath "./src/promptflow/promptflow/_version.py" -Encoding ASCII
|
||||
shell: pwsh
|
||||
env:
|
||||
MSI_PRIVATE_VERSION: ${{ github.event.inputs.set_msi_private_version }}
|
||||
|
||||
- name: Setup and Install dev promptflow
|
||||
if: ${{ github.event.inputs.version == null || github.event.inputs.version == '' }}
|
||||
uses: "./.github/actions/step_sdk_setup"
|
||||
with:
|
||||
setupType: promptflow_with_extra
|
||||
scriptPath: ${{ env.testWorkingDirectory }}
|
||||
|
||||
- name: Setup and Install separate dev promptflow
|
||||
if: ${{ github.event.inputs.version == null || github.event.inputs.version == '' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
|
||||
pip install ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install ${{ github.workspace }}/src/promptflow-core[executor-service,azureml-serving]
|
||||
pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
|
||||
pip install ${{ github.workspace }}/src/promptflow-azure
|
||||
pip install ${{ github.workspace }}/src/promptflow-evals
|
||||
echo "Should fix this after pf-core could install this dependency"
|
||||
pip install azureml-ai-monitoring
|
||||
pip freeze
|
||||
|
||||
- name: Generate promptflow spec file to config pyinstaller
|
||||
working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
|
||||
run: |
|
||||
python generate_dependency.py
|
||||
Get-Content promptflow.spec
|
||||
|
||||
- name: Build Pyinstaller project
|
||||
working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
|
||||
run: |
|
||||
echo 'Version from promptflow: ${{ steps.get-version.outputs.version }}'
|
||||
$text = Get-Content "version_info.txt" -Raw
|
||||
|
||||
$versionString = '${{ steps.get-version.outputs.version }}'
|
||||
$versionArray = $versionString.Split('.')
|
||||
if ($versionArray.Count -ge 4) {
|
||||
$versionArray = $versionArray[0..3]
|
||||
} else {
|
||||
$remainingLength = 4 - $versionArray.Count
|
||||
$zerosToAppend = @(0) * $remainingLength
|
||||
$versionArray += $zerosToAppend
|
||||
}
|
||||
$versionTuple = [string]::Join(', ', $versionArray)
|
||||
$text = $text -replace '\$\((env\.FILE_VERSION)\)', $versionTuple
|
||||
|
||||
$text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
|
||||
$text | Out-File -FilePath "version_info.txt" -Encoding utf8
|
||||
pyinstaller promptflow.spec
|
||||
shell: pwsh
|
||||
|
||||
- name: Generate portable promptflow
|
||||
run: |
|
||||
Compress-Archive -Path ${{ github.workspace }}/scripts/installer/windows/scripts/dist/promptflow -DestinationPath promptflow-${{ steps.get-version.outputs.version }}.zip
|
||||
shell: pwsh
|
||||
|
||||
- name: Build WIX project
|
||||
working-directory: ${{ github.workspace }}/scripts/installer/windows
|
||||
run: |
|
||||
$text = Get-Content "promptflow.wixproj" -Raw
|
||||
$text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
|
||||
$text | Out-File -FilePath "promptflow.wixproj" -Encoding utf8
|
||||
|
||||
$text = Get-Content "product.wxs" -Raw
|
||||
$text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
|
||||
$text | Out-File -FilePath "product.wxs" -Encoding utf8
|
||||
|
||||
msbuild /t:rebuild /p:Configuration=Release /p:Platform=x64 promptflow.wixproj
|
||||
shell: pwsh
|
||||
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
|
||||
- name: Download JSON file from Azure Blob Storage
|
||||
id: download-json
|
||||
run: |
|
||||
az storage blob download --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name ${{ env.AZURE_MSI_CONTAINER_NAME }} --name latest_version.json --file downloaded_version.json --auth-mode login
|
||||
$downloaded_version = (Get-Content downloaded_version.json | ConvertFrom-Json).promptflow
|
||||
echo "::set-output name=downloaded_version::$downloaded_version"
|
||||
|
||||
- name: Check if version input is valid and upload JSON file
|
||||
run: |
|
||||
$version = "${{ steps.get-version.outputs.version }}"
|
||||
$downloaded_version = "${{ steps.download-json.outputs.downloaded_version }}"
|
||||
$uploadAsLatest = "${{ github.event.inputs.uploadAsLatest }}"
|
||||
if ($uploadAsLatest -ieq 'True' -and $version -like '1.*' -and [Version]$version -gt [Version]$downloaded_version){
|
||||
$jsonContent = @{
|
||||
"promptflow" = $version
|
||||
} | ConvertTo-Json -Depth 100
|
||||
$jsonContent | Out-File -FilePath latest_version.json -Encoding UTF8
|
||||
|
||||
Write-Output "Created latest_version.json with version: $version"
|
||||
az storage blob upload --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name ${{ env.AZURE_MSI_CONTAINER_NAME }} --file "latest_version.json" --name "latest_version.json" --overwrite --auth-mode login
|
||||
} else {
|
||||
Write-Output "skip uploading since version input isn't greater than latest version or does not start with '1.'"
|
||||
}
|
||||
|
||||
- name: Upload to Azure Storage
|
||||
run: |
|
||||
function Upload-File($filePath, $blobName, $containerName) {
|
||||
az storage blob upload --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name $containerName --file $filePath --name $blobName --overwrite --auth-mode login
|
||||
}
|
||||
|
||||
$msi_files = Get-ChildItem -Path 'scripts/installer/windows/out/' -Filter *.msi
|
||||
foreach ($msi_file in $msi_files) {
|
||||
if ($env:INPUT_UPLOADASLATEST -ieq 'True') {
|
||||
Upload-File "scripts/installer/windows/out/$($msi_file.Name)" "promptflow.msi" ${{ env.AZURE_MSI_CONTAINER_NAME }}
|
||||
az storage blob copy start --account-name ${{ env.AZURE_ACCOUNT_NAME }} --destination-container ${{ env.AZURE_MSI_CONTAINER_NAME }} --destination-blob "$($msi_file.Name)" --source-container ${{ env.AZURE_MSI_CONTAINER_NAME }} --source-blob "promptflow.msi" --auth-mode login
|
||||
} else {
|
||||
Upload-File "scripts/installer/windows/out/$($msi_file.Name)" "$($msi_file.Name)" ${{ env.AZURE_MSI_CONTAINER_NAME }}
|
||||
}
|
||||
}
|
||||
# Upload zip file
|
||||
if ($env:INPUT_UPLOADASLATEST -ieq 'True') {
|
||||
Upload-File "promptflow-${{ steps.get-version.outputs.version }}.zip" "promptflow.zip" ${{ env.AZURE_PORTABLE_CONTAINER_NAME }}
|
||||
az storage blob copy start --account-name ${{ env.AZURE_ACCOUNT_NAME }} --destination-container ${{ env.AZURE_PORTABLE_CONTAINER_NAME }} --destination-blob "promptflow-${{ steps.get-version.outputs.version }}.zip" --source-container ${{ env.AZURE_PORTABLE_CONTAINER_NAME }} --source-blob "promptflow.zip" --auth-mode login
|
||||
} else {
|
||||
Upload-File "promptflow-${{ steps.get-version.outputs.version }}.zip" "promptflow-${{ steps.get-version.outputs.version }}.zip" ${{ env.AZURE_PORTABLE_CONTAINER_NAME }}
|
||||
}
|
||||
env:
|
||||
INPUT_UPLOADASLATEST: ${{ github.event.inputs.uploadAsLatest }}
|
||||
shell: pwsh
|
||||
@@ -0,0 +1,31 @@
|
||||
name: check_enforcer
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
check_enforcer:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: env | sort >> $GITHUB_OUTPUT
|
||||
- name: Python Setup - ubuntu-latest - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- run: pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: Summarize check status
|
||||
id: summarize_check_status
|
||||
working-directory: ${{ github.workspace }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
python ${{ github.workspace }}/scripts/check_enforcer/check_enforcer.py -t "${{ github.workspace }}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SNIPPET_DEBUG: 1
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Create promptflow release branch
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
# specify when trigger
|
||||
# expected format: 0.1.0b1 or 1.0.0
|
||||
release_version:
|
||||
description: Release version
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
create_release_branch:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: create branch
|
||||
run: |
|
||||
set -x -e
|
||||
|
||||
release_version="${{ inputs.release_version }}"
|
||||
if [[ "$release_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(b[0-9]+)?$ ]]; then
|
||||
echo "configured release version: $release_version"
|
||||
else
|
||||
echo "invalid configured release version: $release_version"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_branch_name="release/promptflow/$release_version"
|
||||
echo "release branch name: $release_branch_name, checking out..."
|
||||
git checkout -b $release_branch_name
|
||||
|
||||
git config --global user.name 'promptflow release'
|
||||
git config --global user.email 'aml-pt-eng@microsoft.com'
|
||||
git push --set-upstream origin $release_branch_name
|
||||
@@ -0,0 +1,87 @@
|
||||
name: Create promptflow release tag
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_version:
|
||||
description: "Release version"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
create_release_tag:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: create branch
|
||||
run: |
|
||||
set -x -e
|
||||
|
||||
branch_name="${{ github.ref_name }}"
|
||||
echo "branch name: $branch_name"
|
||||
|
||||
if [[ $branch_name != release/promptflow/* ]]; then
|
||||
echo "not a release branch for promptflow, exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_version=${branch_name#release/promptflow/}
|
||||
echo "release version: $release_version"
|
||||
|
||||
echo "replacing version in src/promptflow/promptflow/_version.py..."
|
||||
sed -i "s/0.0.1/$release_version/g" src/promptflow/promptflow/_version.py
|
||||
echo "replaced src/promptflow/promptflow/_version.py:"
|
||||
cat src/promptflow/promptflow/_version.py
|
||||
|
||||
if [[ $(git diff --name-only) == *"src/promptflow/promptflow/_version.py"* ]]; then
|
||||
git add src/promptflow/promptflow/_version.py
|
||||
git config --global user.name 'promptflow release'
|
||||
git config --global user.email 'aml-pt-eng@microsoft.com'
|
||||
git commit -m "update version in _version.py for promptflow"
|
||||
git push --set-upstream origin $branch_name
|
||||
fi
|
||||
|
||||
git tag promptflow_$release_version
|
||||
git push origin --tags
|
||||
|
||||
# write environment variable to `GITHUB_ENV` to pass values between steps
|
||||
# https://docs.github.com/en/github-ae@latest/actions/learn-github-actions/variables#passing-values-between-steps-and-jobs-in-a-workflow
|
||||
echo "release_version=$release_version" >> "$GITHUB_ENV"
|
||||
echo "release_tag=$release_tag" >> "$GITHUB_ENV"
|
||||
|
||||
- name: create asset
|
||||
run: |
|
||||
cd src
|
||||
tar -czvf promptflow-$release_version.tar.gz promptflow
|
||||
|
||||
- name: create release note
|
||||
run: |
|
||||
cp ./scripts/release/promptflow-release-note.md ./src/promptflow/release_note.md
|
||||
sed -i "s/{{VERSION}}/$release_version/g" ./src/promptflow/release_note.md
|
||||
cat ./src/promptflow/release_note.md
|
||||
|
||||
- name: create release
|
||||
id: create_release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: promptflow_${{ inputs.release_version }}
|
||||
release_name: promptflow ${{ inputs.release_version }}
|
||||
body_path: ./src/promptflow/release_note.md
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
- name: upload asset
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: ./src/promptflow-${{ inputs.release_version }}.tar.gz
|
||||
asset_name: promptflow-${{ inputs.release_version }}.tar.gz
|
||||
asset_content_type: application/gzip
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Flake8 Lint
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
flake8:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: install dependencies
|
||||
run: pip install flake8
|
||||
|
||||
- name: run flake8
|
||||
run: flake8
|
||||
@@ -0,0 +1,35 @@
|
||||
name: examples_flowdag_schema_check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- examples/**
|
||||
- .github/workflows/flowdag_schema_check.yml
|
||||
- scripts/readme/schema_checker.py
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
jobs:
|
||||
examples_flowdag_schema_check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: env | sort >> $GITHUB_OUTPUT
|
||||
- name: Python Setup - ubuntu-latest - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- run: |
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
- name: Summarize check status
|
||||
id: summarize_check_status
|
||||
working-directory: ${{ github.workspace }}
|
||||
shell: pwsh
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/src/promptflow
|
||||
run: |
|
||||
cd ${{ github.workspace }}/src
|
||||
pip install ./promptflow[azure]
|
||||
pip install ./promptflow-tools
|
||||
python ${{ github.workspace }}/scripts/readme/schema_checker.py
|
||||
@@ -0,0 +1,24 @@
|
||||
name: "Pull Request Labeler"
|
||||
on:
|
||||
- pull_request_target
|
||||
|
||||
jobs:
|
||||
triage:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@v4
|
||||
|
||||
- name: apply a label to pull request from fork
|
||||
if: ${{ github.event.pull_request.head.repo.full_name != 'microsoft/promptflow' }}
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.addLabels({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
labels: ['external']
|
||||
})
|
||||
@@ -0,0 +1,167 @@
|
||||
name: promptflow-core-test [Pure]
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-tracing/**
|
||||
- src/promptflow-core/**
|
||||
- .github/workflows/promptflow-core-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
PF_DISABLE_TRACING: "false"
|
||||
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
core_test:
|
||||
environment:
|
||||
internal
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: |
|
||||
poetry install -E executor-service --with ci,test
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: set test mode
|
||||
run: |
|
||||
echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- name: Azure login (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: generate live test resources (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: generate live test resources (pull_request workflow)
|
||||
if: github.event_name == 'pull_request'
|
||||
working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
run: |
|
||||
cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
|
||||
- name: run core tests
|
||||
run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/core/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/core/htmlcov/
|
||||
|
||||
azureml_serving_test:
|
||||
environment:
|
||||
internal
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: |
|
||||
poetry install -E azureml-serving --with ci,test
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: set test mode
|
||||
run: |
|
||||
echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- name: Azure login (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: generate live test resources (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: generate live test resources (pull_request workflow)
|
||||
if: github.event_name == 'pull_request'
|
||||
working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
run: |
|
||||
cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
|
||||
- name: run azureml-serving tests
|
||||
run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/azureml-serving/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/azureml-serving/htmlcov/
|
||||
|
||||
report:
|
||||
needs: [core_test, azureml_serving_test]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-core test result
|
||||
comment_title: promptflow-core test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
# TODO: Enable coverage check after core test fully setup
|
||||
# - uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
# with:
|
||||
# filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
|
||||
# badge: true
|
||||
# fail_below_min: true
|
||||
# format: markdown
|
||||
# hide_complexity: true
|
||||
# output: both
|
||||
# thresholds: 40 60
|
||||
@@ -0,0 +1,105 @@
|
||||
name: promptflow-evals-e2e-test-azure
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
- .github/workflows/promptflow-evals-e2e-test-azure.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: set test mode
|
||||
# Always run in replay mode for now until we figure out the test resource to run live mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
|
||||
#run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow packages in editable mode
|
||||
run: |
|
||||
poetry run pip install -e ../promptflow
|
||||
poetry run pip install -e ../promptflow-core
|
||||
poetry run pip install -e ../promptflow-devkit
|
||||
poetry run pip install -e ../promptflow-tracing
|
||||
poetry run pip install -e ../promptflow-tools
|
||||
poetry run pip install -e ../promptflow-azure
|
||||
poetry run pip install -e ../promptflow-evals
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
|
||||
- name: run e2e tests
|
||||
id: run_e2e_tests_azure
|
||||
run: |
|
||||
poetry run pytest -m azuretest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
|
||||
poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity e2e_tests_azure --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
report:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-evals test result
|
||||
comment_title: promptflow-evals test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
- uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: false
|
||||
format: markdown
|
||||
hide_complexity: true
|
||||
output: both
|
||||
thresholds: 40 80
|
||||
@@ -0,0 +1,107 @@
|
||||
name: promptflow-evals-e2e-test-local
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
- .github/workflows/promptflow-evals-e2e-test-local.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: set test mode
|
||||
# Always run in replay mode for now until we figure out the test resource to run live mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
|
||||
#run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow packages in editable mode
|
||||
run: |
|
||||
poetry run pip install -e ../promptflow
|
||||
poetry run pip install -e ../promptflow-core
|
||||
poetry run pip install -e ../promptflow-devkit
|
||||
poetry run pip install -e ../promptflow-tracing
|
||||
poetry run pip install -e ../promptflow-tools
|
||||
poetry run pip install -e ../promptflow-evals
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
|
||||
- name: check azure is not installed
|
||||
run: poetry run pytest ../../scripts/code_qa/assert_local_install.py
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run e2e tests
|
||||
id: run_e2e_tests_local
|
||||
run: |
|
||||
poetry run pytest -m localtest tests/evals/e2etests --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
|
||||
poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity e2e_tests_local --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
report:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-evals test result
|
||||
comment_title: promptflow-evals test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
- uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: false
|
||||
format: markdown
|
||||
hide_complexity: true
|
||||
output: both
|
||||
thresholds: 40 80
|
||||
@@ -0,0 +1,61 @@
|
||||
name: promptflow-evals-installation-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
- .github/workflows/promptflow-evals-installation-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: build
|
||||
run: poetry build
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow-evals
|
||||
path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_evals-*.whl
|
||||
|
||||
test:
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: promptflow-evals
|
||||
path: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install virtualenv
|
||||
run: python -m pip install virtualenv
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow-evals from wheel
|
||||
id: install_promptflow_no_extras
|
||||
run: |
|
||||
bash ../../scripts/code_qa/calculate_install_time.sh -r ${{ github.run_id }} -w ${{ github.workflow }} -a ${{ github.action }} -b ${{ github.ref }} -l "300"
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow-evals from wheel
|
||||
id: install_promptflow_with_extras
|
||||
run: |
|
||||
bash ../../scripts/code_qa/calculate_install_time.sh -r ${{ github.run_id }} -w ${{ github.workflow }} -a ${{ github.action }} -b ${{ github.ref }} -e "[azure]" -l "300"
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
@@ -0,0 +1,71 @@
|
||||
name: promptflow-evals-performance-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
- .github/workflows/promptflow-evals-performance-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow-evals from wheel
|
||||
id: install_promptflow
|
||||
run: |
|
||||
# Estimate the installation time.
|
||||
poetry run pip install -e ../promptflow
|
||||
poetry run pip install -e ../promptflow-core
|
||||
poetry run pip install -e ../promptflow-devkit
|
||||
poetry run pip install -e ../promptflow-tracing
|
||||
poetry run pip install -e ../promptflow-tools
|
||||
poetry run pip install -e ../promptflow-azure
|
||||
poetry run pip install -e ../promptflow-evals
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: azure/login@v1
|
||||
with:
|
||||
client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
|
||||
- name: run performance tests
|
||||
id: performance_tests
|
||||
run: |
|
||||
# Estimate the run time for evaluator.
|
||||
poetry run pytest -m performance_test --junit-xml=test-results.xml
|
||||
poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity evaluator_live_tests_run_time_s --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
@@ -0,0 +1,88 @@
|
||||
name: promptflow-evals-unit-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
- .github/workflows/promptflow-evals-unit-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow packages in editable mode
|
||||
run: |
|
||||
poetry run pip install -e ../promptflow
|
||||
poetry run pip install -e ../promptflow-core
|
||||
poetry run pip install -e ../promptflow-devkit
|
||||
poetry run pip install -e ../promptflow-tracing
|
||||
poetry run pip install -e ../promptflow-tools
|
||||
poetry run pip install -e ../promptflow-azure
|
||||
poetry run pip install -e ../promptflow-evals
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run unit tests
|
||||
id: run_unit_tests
|
||||
run: |
|
||||
poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --cov-fail-under=58
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
report:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-evals test result
|
||||
comment_title: promptflow-evals test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
- uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: false
|
||||
format: markdown
|
||||
hide_complexity: true
|
||||
output: both
|
||||
thresholds: 40 60
|
||||
@@ -0,0 +1,173 @@
|
||||
name: promptflow-executor-e2e-test
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 20 * * *" # Every day starting at 4:40 BJT
|
||||
workflow_dispatch:
|
||||
env:
|
||||
packageSetupType: promptflow_with_extra
|
||||
testWorkingDirectory: ${{ github.workspace }}/src/promptflow
|
||||
PYTHONPATH: ${{ github.workspace }}/src/promptflow
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
authorize:
|
||||
environment:
|
||||
# forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
|
||||
${{ github.event_name == 'pull_request_target' &&
|
||||
github.event.pull_request.head.repo.full_name != github.repository &&
|
||||
'external' || 'internal' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: true
|
||||
build:
|
||||
needs: authorize
|
||||
strategy:
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Display and Set Environment Variables
|
||||
run: |
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
id: display_env
|
||||
shell: bash -el {0}
|
||||
- name: Python Setup - ubuntu-latest - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- name: Build wheel
|
||||
uses: "./.github/actions/step_sdk_setup"
|
||||
with:
|
||||
setupType: promptflow_with_extra
|
||||
scriptPath: ${{ env.testWorkingDirectory }}
|
||||
- name: Upload Wheel
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: |
|
||||
${{ github.workspace }}/src/promptflow/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-tools/dist/*.whl
|
||||
|
||||
executor_e2e_tests:
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
environment:
|
||||
internal
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Set test mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Python Setup - ${{ matrix.os }} - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: artifacts
|
||||
- name: Install wheel
|
||||
shell: pwsh
|
||||
working-directory: artifacts
|
||||
run: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
|
||||
pip install ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
|
||||
pip install ${{ github.workspace }}/src/promptflow-devkit
|
||||
pip install ${{ github.workspace }}/src/promptflow-azure
|
||||
gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
|
||||
gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
|
||||
pip freeze
|
||||
- name: install recording
|
||||
run: |
|
||||
pip install vcrpy
|
||||
pip install .
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.testWorkingDirectory }}
|
||||
- name: Get number of CPU cores
|
||||
uses: SimenB/github-actions-cpu-cores@v1
|
||||
id: cpu-cores
|
||||
- name: Run Coverage Test
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
gci env:* | sort-object name
|
||||
az account show
|
||||
pip install langchain-community tenacity<8.4.0
|
||||
# numexpr is required by langchain in e2e tests.
|
||||
pip install numexpr
|
||||
python scripts/building/run_coverage_tests.py `
|
||||
-p ${{ env.testWorkingDirectory }}/promptflow `
|
||||
-t ${{ env.testWorkingDirectory }}/tests/executor/e2etests `
|
||||
-l eastus `
|
||||
-m "all" `
|
||||
-n ${{ steps.cpu-cores.outputs.count }}`
|
||||
--coverage-config ${{ env.testWorkingDirectory }}/tests/executor/.coveragerc `
|
||||
--disable-cov-branch
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python 3.9) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ github.workspace }}/*.xml
|
||||
${{ github.workspace }}/htmlcov/
|
||||
publish-test-results:
|
||||
name: "Publish Tests Results"
|
||||
needs: executor_e2e_tests
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-executor-e2e-test.yml
|
||||
testResultTitle: Executor E2E Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 70
|
||||
context: test/executor_e2e
|
||||
@@ -0,0 +1,175 @@
|
||||
name: promptflow-executor-unit-test
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 19 * * *" # Every day starting at 3:40 BJT
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
env:
|
||||
packageSetupType: promptflow_with_extra
|
||||
testWorkingDirectory: ${{ github.workspace }}/src/promptflow
|
||||
PYTHONPATH: ${{ github.workspace }}/src/promptflow
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
jobs:
|
||||
authorize:
|
||||
environment:
|
||||
# forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
|
||||
${{ github.event_name == 'pull_request_target' &&
|
||||
github.event.pull_request.head.repo.full_name != github.repository &&
|
||||
'external' || 'internal' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: true
|
||||
build:
|
||||
needs: authorize
|
||||
strategy:
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Display and Set Environment Variables
|
||||
run: |
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
id: display_env
|
||||
shell: bash -el {0}
|
||||
- name: Python Setup - ubuntu-latest - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- name: Build wheel
|
||||
uses: "./.github/actions/step_sdk_setup"
|
||||
with:
|
||||
setupType: promptflow_with_extra
|
||||
scriptPath: ${{ env.testWorkingDirectory }}
|
||||
- name: Upload Wheel
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: |
|
||||
${{ github.workspace }}/src/promptflow/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-tools/dist/*.whl
|
||||
executor_unit_tests:
|
||||
needs: build
|
||||
environment:
|
||||
internal
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Set test mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Display and Set Environment Variables
|
||||
run: |
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
id: display_env
|
||||
shell: bash -el {0}
|
||||
- name: Python Setup - ${{ matrix.os }} - Python Version 3.9
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: 3.9
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: artifacts
|
||||
- name: Install wheel
|
||||
shell: pwsh
|
||||
working-directory: artifacts
|
||||
run: |
|
||||
Set-PSDebug -Trace 1
|
||||
pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
|
||||
pip install ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
|
||||
pip install ${{ github.workspace }}/src/promptflow-devkit
|
||||
pip install ${{ github.workspace }}/src/promptflow-azure
|
||||
gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
|
||||
gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
|
||||
pip freeze
|
||||
- name: install recording
|
||||
run: |
|
||||
pip install vcrpy
|
||||
pip install .
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.testWorkingDirectory }}
|
||||
- name: Get number of CPU cores
|
||||
uses: SimenB/github-actions-cpu-cores@v1
|
||||
id: cpu-cores
|
||||
- name: Run Coverage Test
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
gci env:* | sort-object name
|
||||
az account show
|
||||
pip install langchain-community tenacity<8.4.0
|
||||
python scripts/building/run_coverage_tests.py `
|
||||
-p ${{ env.testWorkingDirectory }}/promptflow `
|
||||
-t ${{ env.testWorkingDirectory }}/tests/executor/unittests `
|
||||
-l eastus `
|
||||
-m "all" `
|
||||
-n ${{ steps.cpu-cores.outputs.count }} `
|
||||
--coverage-config ${{ env.testWorkingDirectory }}/tests/executor/.coveragerc `
|
||||
--disable-cov-branch
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python 3.9) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ github.workspace }}/*.xml
|
||||
${{ github.workspace }}/htmlcov/
|
||||
publish-test-results:
|
||||
name: "Publish Tests Results"
|
||||
needs: executor_unit_tests
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-executor-unit-test.yml
|
||||
testResultTitle: Executor Unit Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 50
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
context: test/executor_unit
|
||||
@@ -0,0 +1,126 @@
|
||||
name: promptflow-global-config-test
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 18 * * *" # Every day starting at 2:40 BJT
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
||||
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
|
||||
TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
|
||||
AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
|
||||
jobs:
|
||||
authorize:
|
||||
environment:
|
||||
# forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
|
||||
${{ github.event_name == 'pull_request_target' &&
|
||||
github.event.pull_request.head.repo.full_name != github.repository &&
|
||||
'external' || 'internal' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: true
|
||||
sdk_cli_global_config_tests:
|
||||
needs: authorize
|
||||
environment:
|
||||
internal
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
fetch-depth: 0
|
||||
- name: merge main to current branch
|
||||
uses: "./.github/actions/step_merge_main"
|
||||
- name: Display and Set Environment Variables
|
||||
run: |
|
||||
export pyVersion="3.9"
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
id: display_env
|
||||
shell: bash -el {0}
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ steps.display_env.outputs.pyVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
set -xe
|
||||
poetry install --only test
|
||||
poetry run pip install ${{ env.TRACING_DIRECTORY }}
|
||||
poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
|
||||
poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
|
||||
poetry run pip install -e ${{ env.AZURE_DIRECTORY }}
|
||||
|
||||
echo "Need to install promptflow to avoid tool dependency issue"
|
||||
poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
poetry run pip install ${{ env.TOOL_DIRECTORY }}
|
||||
poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
|
||||
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
poetry run pip show promptflow-devkit
|
||||
poetry run pip show promptflow-azure
|
||||
poetry run pip show promptflow-tools
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Install Azure Login items
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
pip install azure-identity
|
||||
pip install azure-keyvault
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: run devkit tests
|
||||
run: |
|
||||
poetry run pytest ./tests/sdk_cli_global_config_test --cov=promptflow --cov-config=pyproject.toml \
|
||||
--cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
publish-test-results-global-config-test:
|
||||
needs: sdk_cli_global_config_tests
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-global-config-test.yml
|
||||
testResultTitle: SDK CLI Global Config Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 0
|
||||
context: test/sdk_cli
|
||||
@@ -0,0 +1,66 @@
|
||||
name: promptflow-import-linter
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-tracing/**
|
||||
- src/promptflow-core/**
|
||||
- src/promptflow-devkit/**
|
||||
- src/promptflow-azure/**
|
||||
- .github/workflows/promptflow-import-linter.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.11
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Install all packages
|
||||
run: |
|
||||
touch src/promptflow-tracing/promptflow/__init__.py
|
||||
poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing --with dev
|
||||
touch src/promptflow-core/promptflow/__init__.py
|
||||
poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core --with dev
|
||||
touch src/promptflow-devkit/promptflow/__init__.py
|
||||
poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit --with dev
|
||||
touch src/promptflow-azure/promptflow/__init__.py
|
||||
poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure --with dev
|
||||
touch src/promptflow-evals/promptflow/__init__.py
|
||||
poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals --with dev
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: import lint
|
||||
run: |
|
||||
echo "=== Running import lint in promptflow-tracing ==="
|
||||
poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing run lint-imports
|
||||
echo "=== Running import lint in promptflow-core ==="
|
||||
poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core run lint-imports
|
||||
echo "=== Running import lint in promptflow-devkit ==="
|
||||
poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit run lint-imports
|
||||
echo "=== Running import lint in promptflow-azure ==="
|
||||
poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure run lint-imports
|
||||
echo "=== Running import lint in promptflow-evals ==="
|
||||
poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals run lint-imports
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: import lint testing private imports from global
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure
|
||||
run: |
|
||||
set -xe
|
||||
rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing/promptflow/__init__.py
|
||||
rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-core/promptflow/__init__.py
|
||||
rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit/promptflow/__init__.py
|
||||
rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure/promptflow/__init__.py
|
||||
rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals/promptflow/__init__.py
|
||||
echo "=== Add more import linter when facing more import errors ==="
|
||||
|
||||
echo "=== promptflow-azure full lints ==="
|
||||
poetry run pip install langchain
|
||||
poetry run pip install "tenacity<8.4.0"
|
||||
poetry run python ${{ github.workspace }}/scripts/import_linter/import_linter.py
|
||||
@@ -0,0 +1,87 @@
|
||||
name: promptflow-parallel-e2e-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow/**
|
||||
- src/promptflow-core/**
|
||||
- src/promptflow-tracing/**
|
||||
- src/promptflow-parallel/**
|
||||
- .github/workflows/promptflow-parallel-e2e-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-parallel
|
||||
|
||||
jobs:
|
||||
parallel-e2e-test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
environment:
|
||||
internal
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: set test mode
|
||||
# Always run in replay mode for now until we figure out the test resource to run live mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
|
||||
#run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install promptflow packages in editable mode
|
||||
run: |
|
||||
set -xe
|
||||
poetry install --with ci,test
|
||||
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run e2e tests
|
||||
run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
parallel-e2e-test-report:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-parallel-e2e-test.yml
|
||||
testResultTitle: Parallel E2E Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 40
|
||||
context: test/parallel-e2e
|
||||
@@ -0,0 +1,87 @@
|
||||
name: promptflow-parallel-unit-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 10 * * *" # 2:40 PST every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow/**
|
||||
- src/promptflow-core/**
|
||||
- src/promptflow-tracing/**
|
||||
- src/promptflow-parallel/**
|
||||
- .github/workflows/promptflow-parallel-unit-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-parallel
|
||||
|
||||
jobs:
|
||||
parallel-unit-test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
environment:
|
||||
internal
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: set test mode
|
||||
# Always run in replay mode for now until we figure out the test resource to run live mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
|
||||
#run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install promptflow packages in editable mode
|
||||
run: |
|
||||
set -xe
|
||||
poetry install --with ci,test
|
||||
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run unit tests
|
||||
run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
parallel-unit-test-report:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-parallel-unit-test.yml
|
||||
testResultTitle: Parallel Unit Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 40
|
||||
context: test/parallel-unit
|
||||
@@ -0,0 +1,468 @@
|
||||
name: promptflow-release-testing-matrix
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
# can leave empty when trigger manually
|
||||
# GitHub Actions API for trigger does not return workflow run id
|
||||
# there we reference below Stack Overflow solution:
|
||||
# https://stackoverflow.com/a/69500478
|
||||
# which adds an identifier in workflow run jobs and can be used for filter
|
||||
id:
|
||||
description: Identifier for the workflow run
|
||||
required: false
|
||||
type: string
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
|
||||
CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
||||
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
|
||||
TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
id:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: workflow run id - ${{ inputs.id }}
|
||||
run: |
|
||||
echo "workflow run id: ${{ inputs.id }}"
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- uses: snok/install-poetry@v1
|
||||
- working-directory: ${{ env.TRACING_DIRECTORY }}
|
||||
run: poetry build -f wheel
|
||||
- working-directory: ${{ env.CORE_DIRECTORY }}
|
||||
run: poetry build -f wheel
|
||||
- working-directory: ${{ env.DEVKIT_DIRECTORY }}
|
||||
run: poetry build -f wheel
|
||||
- working-directory: ${{ env.AZURE_DIRECTORY }}
|
||||
run: poetry build -f wheel
|
||||
- working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
run: |
|
||||
pip install -r ./dev_requirements.txt
|
||||
python ./setup.py bdist_wheel
|
||||
- working-directory: ${{ env.TOOL_DIRECTORY }}
|
||||
run: python ./setup.py bdist_wheel
|
||||
|
||||
- name: Upload Wheel
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: |
|
||||
${{ github.workspace }}/src/promptflow/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-tracing/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-core/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-devkit/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-azure/dist/*.whl
|
||||
${{ github.workspace }}/src/promptflow-tools/dist/*.whl
|
||||
|
||||
promptflow_tracing_tests:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
||||
needs: build
|
||||
env:
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Display and Set Environment Variables
|
||||
run:
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
||||
- name: install promptflow-tracing from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry install
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run e2e tests
|
||||
run: poetry run pytest -m e2etest --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow_tracing_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
|
||||
|
||||
promptflow_core_tests:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
||||
needs: build
|
||||
environment:
|
||||
internal
|
||||
env:
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: install promptflow-core from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: |
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
|
||||
poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run core tests
|
||||
run: poetry run pytest ./tests/core --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow_core_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
|
||||
promptflow_core_azureml_serving_tests:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
||||
needs: build
|
||||
env:
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
||||
- name: install promptflow-core from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: |
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
||||
poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run azureml-serving tests
|
||||
run: poetry run pytest ./tests/azureml-serving --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow_core_azureml_serving_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
|
||||
|
||||
promptflow_devkit_tests:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
||||
needs: build
|
||||
environment:
|
||||
internal
|
||||
env:
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: install promptflow-devkit from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: |
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run devkit tests
|
||||
run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest" --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow_devkit_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
|
||||
|
||||
promptflow_azure_tests:
|
||||
needs: build
|
||||
environment:
|
||||
internal
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
env:
|
||||
PROMPT_FLOW_TEST_MODE: "live"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
|
||||
PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: ${{ env.WORKING_DIRECTORY }}/artifacts
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: install promptflow-azure from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: |
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_azure-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
|
||||
poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
|
||||
poetry run pip install -e ../promptflow-recording
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run azure tests
|
||||
run: poetry run pytest ./tests/sdk_cli_azure_test -n auto -m "unittest or e2etest" --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow_azure_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
|
||||
|
||||
promptflow_executor_tests:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
|
||||
needs: build
|
||||
environment:
|
||||
internal
|
||||
env:
|
||||
testWorkingDirectory: src/promptflow
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Display and Set Environment Variables
|
||||
run:
|
||||
env | sort >> $GITHUB_OUTPUT
|
||||
shell: bash -el {0}
|
||||
- name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
with:
|
||||
pythonVersion: ${{ matrix.pythonVersion }}
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: wheel
|
||||
path: artifacts
|
||||
- name: install recording
|
||||
run: |
|
||||
pip install vcrpy
|
||||
pip install -e .
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Generate Configs
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.testWorkingDirectory }}
|
||||
- name: Install pf
|
||||
shell: pwsh
|
||||
working-directory: artifacts
|
||||
run: |
|
||||
pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
|
||||
pip uninstall -y promptflow-core promptflow-devkit promptflow-tracing
|
||||
pip install ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install ${{ github.workspace }}/src/promptflow-core
|
||||
pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
|
||||
pip install ${{ github.workspace }}/src/promptflow-azure
|
||||
gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
|
||||
gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
|
||||
pip freeze
|
||||
- name: Run Executor Unit Test
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
pip install langchain
|
||||
pip install numexpr
|
||||
python scripts/building/run_coverage_tests.py `
|
||||
-p ${{ github.workspace }}/src/promptflow/promptflow `
|
||||
-t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
|
||||
-l eastus `
|
||||
-m "all" `
|
||||
-o "${{ github.workspace }}/test-results-executor-unit.xml"
|
||||
- name: Run Executor E2E Test
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
pip install langchain langchain_community
|
||||
pip install numexpr
|
||||
python scripts/building/run_coverage_tests.py `
|
||||
-p ${{ github.workspace }}/src/promptflow/promptflow `
|
||||
-t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
|
||||
-l eastus `
|
||||
-m "all" `
|
||||
-o "${{ github.workspace }}/test-results-executor-e2e.xml"
|
||||
- name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
|
||||
path: ${{ github.workspace }}/*.xml
|
||||
|
||||
|
||||
publish-test-results:
|
||||
name: "Publish Tests Results"
|
||||
needs: [ promptflow_devkit_tests, promptflow_azure_tests, promptflow_executor_tests, promptflow_core_tests, promptflow_core_azureml_serving_tests ]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
path: artifacts
|
||||
- name: Publish Test Results
|
||||
uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
files: "artifacts/**/test-*.xml"
|
||||
@@ -0,0 +1,124 @@
|
||||
name: promptflow-sdk-cli-test
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 18 * * *" # Every day starting at 2:40 BJT
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-core/**
|
||||
- src/promptflow-devkit/**
|
||||
- src/promptflow/**
|
||||
- src/promptflow-tracing/**
|
||||
- scripts/building/**
|
||||
- .github/workflows/promptflow-sdk-cli-test.yml
|
||||
- src/promptflow-recording/**
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filepath:
|
||||
description: file or paths you want to trigger a test
|
||||
required: true
|
||||
default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
|
||||
type: string
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
PF_DISABLE_TRACING: "false"
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
|
||||
PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
|
||||
TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
|
||||
jobs:
|
||||
sdk_cli_tests:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
pythonVersion: ['3.9', '3.10', '3.11']
|
||||
environment:
|
||||
internal
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: set test mode
|
||||
run: |
|
||||
echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.pythonVersion }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install test dependency group
|
||||
run: |
|
||||
set -xe
|
||||
poetry install -E pyarrow --with ci,test
|
||||
|
||||
poetry run pip show promptflow-tracing
|
||||
poetry run pip show promptflow-core
|
||||
poetry run pip show promptflow-devkit
|
||||
poetry run pip show promptflow-tools
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: Azure login (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: generate live test resources (non pull_request workflow)
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: "./.github/actions/step_generate_configs"
|
||||
with:
|
||||
targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
- name: generate live test resources (pull_request workflow)
|
||||
if: github.event_name == 'pull_request'
|
||||
working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
|
||||
run: |
|
||||
cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
|
||||
- name: run devkit tests
|
||||
run: |
|
||||
poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
|
||||
--cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
|
||||
--ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: Upload Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
|
||||
- run: poetry install -E executable
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run devkit executable tests
|
||||
run: |
|
||||
poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
|
||||
|
||||
publish-test-results-sdk-cli-test:
|
||||
needs: sdk_cli_tests
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Publish Test Results
|
||||
uses: "./.github/actions/step_publish_test_results"
|
||||
with:
|
||||
testActionFileName: promptflow-sdk-cli-test.yml
|
||||
testResultTitle: SDK CLI Test Result
|
||||
osVersion: ubuntu-latest
|
||||
pythonVersion: 3.9
|
||||
coverageThreshold: 40
|
||||
context: test/sdk_cli
|
||||
@@ -0,0 +1,109 @@
|
||||
name: promptflow-tracing-e2e-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-tracing/**
|
||||
- .github/workflows/promptflow-tracing-e2e-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
PF_DISABLE_TRACING: "false"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: build
|
||||
run: poetry build
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow-tracing
|
||||
path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
|
||||
|
||||
tracing-e2e-test:
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: set test mode
|
||||
run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: promptflow-tracing
|
||||
path: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow-tracing from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: poetry run pip install $(python -c "import glob; print(glob.glob('promptflow_tracing-*.whl')[0])")
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry install
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: generate end-to-end test config from secret
|
||||
run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: run e2e tests
|
||||
run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
report:
|
||||
needs: tracing-e2e-test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-tracing test result
|
||||
comment_title: promptflow-tracing test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
- uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: true
|
||||
format: markdown
|
||||
hide_complexity: true
|
||||
output: both
|
||||
thresholds: 40 80
|
||||
@@ -0,0 +1,100 @@
|
||||
name: promptflow-tracing-unit-test
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-tracing/**
|
||||
- .github/workflows/promptflow-tracing-unit-test.yml
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
PF_DISABLE_TRACING: "false"
|
||||
WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
|
||||
RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: build
|
||||
run: poetry build
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: promptflow-tracing
|
||||
path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
|
||||
|
||||
tracing-unit-test:
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-13]
|
||||
python-version: ['3.9', '3.10', '3.11']
|
||||
fail-fast: false
|
||||
# snok/install-poetry need this to support Windows
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: snok/install-poetry@v1
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: promptflow-tracing
|
||||
path: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install promptflow-tracing from wheel
|
||||
# wildcard expansion (*) does not work in Windows, so leverage python to find and install
|
||||
run: poetry run pip install $(python -c "import glob; print(glob.glob('promptflow_tracing-*.whl')[0])")
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install test dependency group
|
||||
run: poetry install --only test
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: install recording
|
||||
run: poetry install
|
||||
working-directory: ${{ env.RECORD_DIRECTORY }}
|
||||
- name: run unit tests
|
||||
run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
- name: upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: report-${{ matrix.os }}-py${{ matrix.python-version }}
|
||||
path: |
|
||||
${{ env.WORKING_DIRECTORY }}/*.xml
|
||||
${{ env.WORKING_DIRECTORY }}/htmlcov/
|
||||
|
||||
report:
|
||||
needs: tracing-unit-test
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
checks: write
|
||||
pull-requests: write
|
||||
contents: read
|
||||
issues: read
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
- uses: EnricoMi/publish-unit-test-result-action@v2
|
||||
with:
|
||||
check_name: promptflow-tracing test result
|
||||
comment_title: promptflow-tracing test result
|
||||
files: "artifacts/**/test-results.xml" # align with `--junit-xml` in pyproject.toml
|
||||
- uses: irongut/CodeCoverageSummary@v1.3.0
|
||||
with:
|
||||
filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
|
||||
badge: true
|
||||
fail_below_min: true
|
||||
format: markdown
|
||||
hide_complexity: true
|
||||
output: both
|
||||
thresholds: 40 60
|
||||
@@ -0,0 +1,87 @@
|
||||
name: Publish Promptflow Doc
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- preview/docs
|
||||
paths:
|
||||
- 'README.md'
|
||||
- 'docs/**'
|
||||
- 'examples/**.ipynb'
|
||||
- 'scripts/docs/**'
|
||||
- '.github/workflows/publish_doc.yml'
|
||||
- 'src/promptflow-tracing/promptflow/**'
|
||||
- 'src/promptflow-core/promptflow/**'
|
||||
- 'src/promptflow-devkit/promptflow/**'
|
||||
- 'src/promptflow-azure/promptflow/**'
|
||||
- 'src/promptflow-rag/promptflow/**'
|
||||
- 'src/promptflow-evals/promptflow/**'
|
||||
|
||||
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Allow one concurrent deployment
|
||||
concurrency:
|
||||
group: "pages"
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
packageSetupType: promptflow_with_extra
|
||||
testWorkingDirectory: ${{ github.workspace }}/src/promptflow
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: windows-latest
|
||||
name: Build
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Python Setup
|
||||
uses: "./.github/actions/step_create_python_environment"
|
||||
|
||||
- name: Install packages
|
||||
shell: pwsh
|
||||
# Note: Use -e to avoid duplicate object warning when build apidoc.
|
||||
run: |
|
||||
pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-tracing
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-core
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-devkit
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-azure
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-rag
|
||||
pip install -e ${{ github.workspace }}/src/promptflow-evals
|
||||
pip freeze
|
||||
|
||||
- name: Build Doc
|
||||
shell: powershell
|
||||
working-directory: scripts/docs/
|
||||
run: |-
|
||||
pip install langchain
|
||||
./doc_generation.ps1 -WithReferenceDoc:$true
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
# Upload entire repository
|
||||
path: scripts/docs/_build
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
name: Deploy
|
||||
needs: build
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
@@ -0,0 +1,32 @@
|
||||
name: Pylint
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- src/promptflow-evals/**
|
||||
|
||||
jobs:
|
||||
run_pylint:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- uses: snok/install-poetry@v1
|
||||
- name: install pylint and azure-pylint-guidelines-checker
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
set -xe
|
||||
poetry install -C src/promptflow-evals --with dev
|
||||
poetry show -C src/promptflow-evals
|
||||
- name: run pylint
|
||||
working-directory: ${{ env.WORKING_DIRECTORY }}
|
||||
run: |
|
||||
cd src/promptflow-evals
|
||||
poetry run pylint promptflow/evals --rcfile=../../pylintrc
|
||||
@@ -0,0 +1,78 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_configuration
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 22 * * *" # Every day starting at 6:40 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/configuration.ipynb, .github/workflows/samples_configuration.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_configuration:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Generate config.json for canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
|
||||
- name: Generate config.json for production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: Create Aoai Connection
|
||||
run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 1
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Fetch OID token every 4 mins
|
||||
shell: bash
|
||||
run: |
|
||||
while true; do
|
||||
token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
|
||||
az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
|
||||
# Sleep for 4 minutes
|
||||
sleep 240
|
||||
done &
|
||||
- name: Test Notebook
|
||||
working-directory: examples
|
||||
run: |
|
||||
papermill -k python configuration.ipynb configuration.output.ipynb
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: artifact
|
||||
path: examples
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_connections
|
||||
on:
|
||||
schedule:
|
||||
- cron: "40 20 * * *" # Every day starting at 4:40 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/connections/**, examples/*requirements.txt, .github/workflows/samples_connections.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_connections:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/connections/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/connections/README.md -o examples/connections
|
||||
- name: Cat script
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/connections/bash_script.sh
|
||||
@@ -0,0 +1,88 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_connections_connection
|
||||
on:
|
||||
schedule:
|
||||
- cron: "13 22 * * *" # Every day starting at 6:13 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/connections/**, examples/*requirements.txt, .github/workflows/samples_connections_connection.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_connections_connection:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: setup .env file
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create Aoai Connection
|
||||
run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 1
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Fetch OID token every 4 mins
|
||||
shell: bash
|
||||
run: |
|
||||
while true; do
|
||||
token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
|
||||
az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
|
||||
# Sleep for 4 minutes
|
||||
sleep 240
|
||||
done &
|
||||
- name: Test Notebook
|
||||
working-directory: examples/connections
|
||||
run: |
|
||||
papermill -k python connection.ipynb connection.output.ipynb
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/connections
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_basic
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 20 * * *" # Every day starting at 4:30 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_basic.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_basic:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/basic/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/basic/README.md -o examples/flex-flows/basic
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/basic/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_chat_async_stream
|
||||
on:
|
||||
schedule:
|
||||
- cron: "59 20 * * *" # Every day starting at 4:59 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-async-stream/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_async_stream.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_chat_async_stream:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/chat-async-stream/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-async-stream/README.md -o examples/flex-flows/chat-async-stream
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-async-stream/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_chat_basic
|
||||
on:
|
||||
schedule:
|
||||
- cron: "9 20 * * *" # Every day starting at 4:9 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_basic.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_chat_basic:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/chat-basic/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-basic/README.md -o examples/flex-flows/chat-basic
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/chat-basic
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-basic/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_chat_minimal
|
||||
on:
|
||||
schedule:
|
||||
- cron: "29 22 * * *" # Every day starting at 6:29 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-minimal/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_minimal.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_chat_minimal:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/chat-minimal/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-minimal/README.md -o examples/flex-flows/chat-minimal
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/chat-minimal
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-minimal/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_chat_stream
|
||||
on:
|
||||
schedule:
|
||||
- cron: "7 19 * * *" # Every day starting at 3:7 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-stream/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_stream.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_chat_stream:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/chat-stream/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-stream/README.md -o examples/flex-flows/chat-stream
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/chat-stream
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-stream/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_chat_with_functions
|
||||
on:
|
||||
schedule:
|
||||
- cron: "51 21 * * *" # Every day starting at 5:51 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-with-functions/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_with_functions.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_chat_with_functions:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/chat-with-functions/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-with-functions/README.md -o examples/flex-flows/chat-with-functions
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/chat-with-functions
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-with-functions/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_eval_checklist
|
||||
on:
|
||||
schedule:
|
||||
- cron: "56 22 * * *" # Every day starting at 6:56 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/eval-checklist/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_checklist.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_eval_checklist:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/eval-checklist/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-checklist/README.md -o examples/flex-flows/eval-checklist
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/eval-checklist
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/eval-checklist/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_eval_code_quality
|
||||
on:
|
||||
schedule:
|
||||
- cron: "6 22 * * *" # Every day starting at 6:6 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/eval-code-quality/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_code_quality.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_eval_code_quality:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/eval-code-quality/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-code-quality/README.md -o examples/flex-flows/eval-code-quality
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/eval-code-quality
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/eval-code-quality/bash_script.sh
|
||||
@@ -0,0 +1,123 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flex_flows_eval_criteria_with_langchain
|
||||
on:
|
||||
schedule:
|
||||
- cron: "21 20 * * *" # Every day starting at 4:21 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/eval-criteria-with-langchain/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_criteria_with_langchain.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flex_flows_eval_criteria_with_langchain:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
if [[ -e requirements.txt ]]; then
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
- name: Prepare dev requirements
|
||||
working-directory: examples
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r dev_requirements.txt
|
||||
- name: Refine .env file
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create run.yml
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
gpt_base=$(echo ${gpt_base//\//\\/})
|
||||
if [[ -e run.yml ]]; then
|
||||
sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
|
||||
fi
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 0
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Extract Steps examples/flex-flows/eval-criteria-with-langchain/README.md
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-criteria-with-langchain/README.md -o examples/flex-flows/eval-criteria-with-langchain
|
||||
- name: Cat script
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
cat bash_script.sh
|
||||
- name: Run scripts against canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
|
||||
bash bash_script.sh
|
||||
- name: Run scripts against production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
|
||||
export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
|
||||
export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
|
||||
export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
|
||||
bash bash_script.sh
|
||||
- name: Pip List for Debug
|
||||
if : ${{ always() }}
|
||||
working-directory: examples/flex-flows/eval-criteria-with-langchain
|
||||
run: |
|
||||
pip list
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/eval-criteria-with-langchain/bash_script.sh
|
||||
@@ -0,0 +1,88 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flexflows_basic_flexflowquickstart
|
||||
on:
|
||||
schedule:
|
||||
- cron: "55 20 * * *" # Every day starting at 4:55 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_basic_flexflowquickstart.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flexflows_basic_flexflowquickstart:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: setup .env file
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create Aoai Connection
|
||||
run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 1
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Fetch OID token every 4 mins
|
||||
shell: bash
|
||||
run: |
|
||||
while true; do
|
||||
token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
|
||||
az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
|
||||
# Sleep for 4 minutes
|
||||
sleep 240
|
||||
done &
|
||||
- name: Test Notebook
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
papermill -k python flex-flow-quickstart.ipynb flex-flow-quickstart.output.ipynb
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/basic
|
||||
@@ -0,0 +1,78 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flexflows_basic_flexflowquickstartazure
|
||||
on:
|
||||
schedule:
|
||||
- cron: "10 22 * * *" # Every day starting at 6:10 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_basic_flexflowquickstartazure.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flexflows_basic_flexflowquickstartazure:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Generate config.json for canary workspace (scheduled runs only)
|
||||
if: github.event_name == 'schedule'
|
||||
run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
|
||||
- name: Generate config.json for production workspace
|
||||
if: github.event_name != 'schedule'
|
||||
run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: Create Aoai Connection
|
||||
run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 1
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Fetch OID token every 4 mins
|
||||
shell: bash
|
||||
run: |
|
||||
while true; do
|
||||
token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
|
||||
az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
|
||||
# Sleep for 4 minutes
|
||||
sleep 240
|
||||
done &
|
||||
- name: Test Notebook
|
||||
working-directory: examples/flex-flows/basic
|
||||
run: |
|
||||
papermill -k python flex-flow-quickstart-azure.ipynb flex-flow-quickstart-azure.output.ipynb
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/basic
|
||||
@@ -0,0 +1,88 @@
|
||||
# This code is autogenerated.
|
||||
# Code is generated by running custom script: python3 readme.py
|
||||
# Any manual changes to this file may cause incorrect behavior.
|
||||
# Any manual changes will be overwritten if the code is regenerated.
|
||||
|
||||
name: samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow
|
||||
on:
|
||||
schedule:
|
||||
- cron: "11 20 * * *" # Every day starting at 4:11 BJT
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ examples/flex-flows/chat-async-stream/**, examples/*requirements.txt, .github/workflows/samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow.yml ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
IS_IN_CI_PIPELINE: "true"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
internal
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup Python 3.9 environment
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Prepare requirements
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r ${{ github.workspace }}/examples/requirements.txt
|
||||
pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
|
||||
- name: setup .env file
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
|
||||
AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
|
||||
AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
|
||||
if [[ -e .env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
|
||||
mv .env.example .env
|
||||
fi
|
||||
if [[ -e ../.env.example ]]; then
|
||||
echo "env replacement"
|
||||
sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
|
||||
mv ../.env.example ../.env
|
||||
fi
|
||||
- name: Create Aoai Connection
|
||||
run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
|
||||
- name: Random Wait
|
||||
uses: AliSajid/random-wait-action@main
|
||||
with:
|
||||
minimum: 1
|
||||
maximum: 99
|
||||
- name: Azure Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
|
||||
tenant-id: ${{secrets.AZURE_TENANT_ID}}
|
||||
client-id: ${{secrets.AZURE_CLIENT_ID}}
|
||||
- name: Fetch OID token every 4 mins
|
||||
shell: bash
|
||||
run: |
|
||||
while true; do
|
||||
token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
|
||||
token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
|
||||
token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
|
||||
az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
|
||||
# Sleep for 4 minutes
|
||||
sleep 240
|
||||
done &
|
||||
- name: Test Notebook
|
||||
working-directory: examples/flex-flows/chat-async-stream
|
||||
run: |
|
||||
papermill -k python chat-stream-with-async-flex-flow.ipynb chat-stream-with-async-flex-flow.output.ipynb
|
||||
- name: Upload artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: artifact
|
||||
path: examples/flex-flows/chat-async-stream
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user