chore: import upstream snapshot with attribution

.github/workflows/build_doc_ci.yml

@@ -0,0 +1,89 @@
+name: Build Doc CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'README.md'
+      - 'docs/**'
+      - 'examples/**.ipynb'
+      - 'scripts/docs/**'
+      - '.github/workflows/build_doc_ci.yml'
+      - 'src/promptflow-tracing/promptflow/**'
+      - 'src/promptflow-core/promptflow/**'
+      - 'src/promptflow-devkit/promptflow/**'
+      - 'src/promptflow-azure/promptflow/**'
+      - 'src/promptflow-rag/promptflow/**'
+      - 'src/promptflow-evals/promptflow/**'
+
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+
+jobs:
+  build_doc_job:
+    runs-on: windows-latest
+    name: Build Doc
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
+          pip freeze
+
+      - name: Build doc with reference doc
+        shell: powershell
+        working-directory: scripts/docs/
+        run: |-
+          pip install langchain tenacity<8.4.0
+          ./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true 
+
+  # Note: We have this job separately because some error may missing when build link check exists.
+  link_check_job:
+    runs-on: windows-latest
+    name: Build Link Check
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
+          pip freeze
+
+      - name: Build LinkCheck
+        shell: powershell
+        working-directory: scripts/docs/
+        run: |-
+          pip install langchain tenacity<8.4.0
+          ./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true -BuildLinkCheck

.github/workflows/build_msi_installer.yml

@@ -0,0 +1,238 @@
+name: Build and Package MSI & Portable Installer
+
+on:
+  workflow_dispatch:
+    inputs:
+      uploadAsLatest:
+        type: string
+        default: "False"
+        required: false
+        description: 'Also upload the msi installer to storage account as latest'
+
+      version:
+        type: string
+        default: ""
+        required: false
+        description: 'Version of promptflow to install (optional). Will build locally if not specified.'
+
+      set_msi_private_version:
+        type: string
+        default: ""
+        required: false
+        description: 'Set the version of the private msi installer'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: src/promptflow
+  AZURE_ACCOUNT_NAME: promptflowartifact
+  AZURE_MSI_CONTAINER_NAME: msi-installer
+  AZURE_PORTABLE_CONTAINER_NAME: portable-installer
+
+jobs:
+  build_msi_installer:
+    runs-on: windows-latest
+    name: Build Windows MSI
+    environment:
+      internal
+    steps:
+      - name: Check input parameters
+        run: |
+          echo "uploadAsLatest: ${{ inputs.uploadAsLatest }}"
+          echo "version: ${{ inputs.version }}"
+          echo "set_msi_private_version: ${{ inputs.set_msi_private_version }}"
+
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Add msbuild to PATH
+        uses: microsoft/setup-msbuild@v1.1
+
+      - name: Install WIX Toolset
+        shell:  pwsh
+        working-directory: ${{ github.workspace }}/scripts/installer/windows
+        run: |
+          Invoke-WebRequest -Uri 'https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip' -OutFile 'wix-archive.zip'
+          Expand-Archive -Path 'wix-archive.zip' -DestinationPath 'wix'
+          Remove-Item -Path 'wix-archive.zip'
+
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install stable promptflow
+        if: ${{ github.event.inputs.version != null && github.event.inputs.version != '' }}
+        run: |
+          pip install "promptflow[azure,executable,azureml-serving,executor-service]==$env:INPUT_VERSION" promptflow-tools
+          echo "There's no promptflow-evals in pypi, we need to install it from source for now"
+          pip install ${{ github.workspace }}/src/promptflow-evals
+        env:
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+        shell: pwsh
+
+      - name: Get promptflow version
+        id: get-version
+        # Convert string to int since the version tuple used in "version_info" can't start with 0.
+        run: |
+          if ($env:INPUT_VERSION) {
+            $version=$env:INPUT_VERSION
+            $run_version=$(python -c "import promptflow; print(promptflow.__version__)")
+            if ($version -ne $run_version) {
+              throw "Version input does not match the version in promptflow package. Version input: $version, version in promptflow package: $run_version"
+            }
+          }
+          elseif ($env:MSI_PRIVATE_VERSION) {
+            $version=$env:MSI_PRIVATE_VERSION
+          }
+          else {
+            $prefix = 0
+            $year = [int](Get-Date -Format "yy")
+            $monthday = [int](Get-Date -Format "MMdd")
+            $hourminutesecond = [int](Get-Date -Format "HHmmss")
+            $version="$prefix.$year.$monthday.$hourminutesecond"
+          }
+          echo "::set-output name=version::$version"
+        env:
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+          MSI_PRIVATE_VERSION: ${{ github.event.inputs.set_msi_private_version }}
+        shell: pwsh
+
+      - name: Update promptflow package version when set msi private version
+        if: ${{ github.event.inputs.set_msi_private_version != null && github.event.inputs.set_msi_private_version != '' }}
+        run: |
+          $override_version = 'VERSION = "{0}"' -f $env:MSI_PRIVATE_VERSION
+          Write-Host "'$override_version' as version"
+          $override_version | Out-File -FilePath "./src/promptflow/promptflow/_version.py" -Encoding ASCII
+        shell: pwsh
+        env:
+          MSI_PRIVATE_VERSION: ${{ github.event.inputs.set_msi_private_version }}
+
+      - name: Setup and Install dev promptflow
+        if: ${{ github.event.inputs.version == null || github.event.inputs.version == '' }}
+        uses: "./.github/actions/step_sdk_setup"
+        with:
+          setupType: promptflow_with_extra
+          scriptPath: ${{ env.testWorkingDirectory }}
+
+      - name: Setup and Install separate dev promptflow
+        if: ${{ github.event.inputs.version == null || github.event.inputs.version == '' }}
+        shell: pwsh
+        run: |
+          Set-PSDebug -Trace 1
+          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core[executor-service,azureml-serving]
+          pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
+          pip install ${{ github.workspace }}/src/promptflow-azure
+          pip install ${{ github.workspace }}/src/promptflow-evals
+          echo "Should fix this after pf-core could install this dependency"
+          pip install azureml-ai-monitoring
+          pip freeze
+
+      - name: Generate promptflow spec file to config pyinstaller
+        working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
+        run: |
+          python generate_dependency.py
+          Get-Content promptflow.spec
+
+      - name: Build Pyinstaller project
+        working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
+        run: |
+          echo 'Version from promptflow: ${{ steps.get-version.outputs.version }}'
+          $text = Get-Content "version_info.txt" -Raw
+
+          $versionString = '${{ steps.get-version.outputs.version }}'
+          $versionArray = $versionString.Split('.')
+          if ($versionArray.Count -ge 4) {
+              $versionArray = $versionArray[0..3]
+          } else {
+              $remainingLength = 4 - $versionArray.Count
+              $zerosToAppend = @(0) * $remainingLength
+              $versionArray += $zerosToAppend
+          }
+          $versionTuple = [string]::Join(', ', $versionArray)
+          $text = $text -replace '\$\((env\.FILE_VERSION)\)', $versionTuple
+
+          $text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
+          $text | Out-File -FilePath "version_info.txt" -Encoding utf8
+          pyinstaller promptflow.spec
+        shell: pwsh
+
+      - name: Generate portable promptflow
+        run: |
+          Compress-Archive -Path ${{ github.workspace }}/scripts/installer/windows/scripts/dist/promptflow -DestinationPath promptflow-${{ steps.get-version.outputs.version }}.zip
+        shell: pwsh
+
+      - name: Build WIX project
+        working-directory: ${{ github.workspace }}/scripts/installer/windows
+        run: |
+          $text = Get-Content "promptflow.wixproj" -Raw
+          $text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
+          $text | Out-File -FilePath "promptflow.wixproj" -Encoding utf8
+
+          $text = Get-Content "product.wxs" -Raw
+          $text = $text -replace '\$\((env\.CLI_VERSION)\)', '${{ steps.get-version.outputs.version }}'
+          $text | Out-File -FilePath "product.wxs" -Encoding utf8
+
+          msbuild /t:rebuild /p:Configuration=Release /p:Platform=x64 promptflow.wixproj
+        shell: pwsh
+
+      - uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+
+      - name: Download JSON file from Azure Blob Storage
+        id: download-json
+        run: |
+          az storage blob download --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name ${{ env.AZURE_MSI_CONTAINER_NAME }} --name latest_version.json --file downloaded_version.json --auth-mode login
+          $downloaded_version = (Get-Content downloaded_version.json | ConvertFrom-Json).promptflow
+          echo "::set-output name=downloaded_version::$downloaded_version"
+
+      - name: Check if version input is valid and upload JSON file
+        run: |
+          $version = "${{ steps.get-version.outputs.version }}"
+          $downloaded_version = "${{ steps.download-json.outputs.downloaded_version }}"
+          $uploadAsLatest = "${{ github.event.inputs.uploadAsLatest }}"
+          if ($uploadAsLatest -ieq 'True' -and $version -like '1.*' -and [Version]$version -gt [Version]$downloaded_version){
+            $jsonContent = @{
+              "promptflow" = $version
+            } | ConvertTo-Json -Depth 100
+            $jsonContent | Out-File -FilePath latest_version.json -Encoding UTF8
+
+            Write-Output "Created latest_version.json with version: $version"
+            az storage blob upload --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name ${{ env.AZURE_MSI_CONTAINER_NAME }} --file "latest_version.json" --name "latest_version.json" --overwrite --auth-mode login
+          } else {
+            Write-Output "skip uploading since version input isn't greater than latest version or does not start with '1.'"
+          }
+
+      - name: Upload to Azure Storage
+        run: |
+          function Upload-File($filePath, $blobName, $containerName) {
+            az storage blob upload --account-name ${{ env.AZURE_ACCOUNT_NAME }} --container-name $containerName --file $filePath --name $blobName --overwrite --auth-mode login
+          }
+    
+          $msi_files = Get-ChildItem -Path 'scripts/installer/windows/out/' -Filter *.msi
+          foreach ($msi_file in $msi_files) {
+            if ($env:INPUT_UPLOADASLATEST -ieq 'True') {
+              Upload-File "scripts/installer/windows/out/$($msi_file.Name)" "promptflow.msi" ${{ env.AZURE_MSI_CONTAINER_NAME }}
+              az storage blob copy start --account-name ${{ env.AZURE_ACCOUNT_NAME }} --destination-container ${{ env.AZURE_MSI_CONTAINER_NAME }} --destination-blob "$($msi_file.Name)" --source-container ${{ env.AZURE_MSI_CONTAINER_NAME }} --source-blob "promptflow.msi" --auth-mode login
+            } else {
+              Upload-File "scripts/installer/windows/out/$($msi_file.Name)" "$($msi_file.Name)" ${{ env.AZURE_MSI_CONTAINER_NAME }}
+            }
+          }
+          # Upload zip file
+          if ($env:INPUT_UPLOADASLATEST -ieq 'True') {
+            Upload-File "promptflow-${{ steps.get-version.outputs.version }}.zip" "promptflow.zip" ${{ env.AZURE_PORTABLE_CONTAINER_NAME }}
+            az storage blob copy start --account-name ${{ env.AZURE_ACCOUNT_NAME }} --destination-container ${{ env.AZURE_PORTABLE_CONTAINER_NAME }} --destination-blob "promptflow-${{ steps.get-version.outputs.version }}.zip" --source-container ${{ env.AZURE_PORTABLE_CONTAINER_NAME }} --source-blob "promptflow.zip" --auth-mode login
+          } else {
+            Upload-File "promptflow-${{ steps.get-version.outputs.version }}.zip" "promptflow-${{ steps.get-version.outputs.version }}.zip" ${{ env.AZURE_PORTABLE_CONTAINER_NAME }}
+          }
+        env:
+          INPUT_UPLOADASLATEST: ${{ github.event.inputs.uploadAsLatest }}
+        shell: pwsh

.github/workflows/check_enforcer.yml

@@ -0,0 +1,31 @@
+name: check_enforcer
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check_enforcer:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: env | sort >> $GITHUB_OUTPUT
+      - name: Python Setup - ubuntu-latest - Python Version 3.9
+        uses: "./.github/actions/step_create_python_environment"
+        with:
+          pythonVersion: 3.9
+      - run: pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Summarize check status
+        id: summarize_check_status
+        working-directory: ${{ github.workspace }}
+        shell: pwsh
+        run: |
+            python ${{ github.workspace }}/scripts/check_enforcer/check_enforcer.py -t "${{ github.workspace }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SNIPPET_DEBUG: 1

.github/workflows/create_promptflow_release_branch.yml

@@ -0,0 +1,39 @@
+name: Create promptflow release branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      # specify when trigger
+      # expected format: 0.1.0b1 or 1.0.0
+      release_version:
+        description: Release version
+        required: true
+        type: string
+
+jobs:
+  create_release_branch:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: create branch
+        run: |
+          set -x -e
+
+          release_version="${{ inputs.release_version }}"
+          if [[ "$release_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(b[0-9]+)?$ ]]; then
+            echo "configured release version: $release_version"
+          else
+            echo "invalid configured release version: $release_version"
+            exit 1
+          fi
+
+          release_branch_name="release/promptflow/$release_version"
+          echo "release branch name: $release_branch_name, checking out..."
+          git checkout -b $release_branch_name
+
+          git config --global user.name 'promptflow release'
+          git config --global user.email 'aml-pt-eng@microsoft.com'
+          git push --set-upstream origin $release_branch_name

.github/workflows/create_promptflow_release_tag.yml

@@ -0,0 +1,87 @@
+name: Create promptflow release tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_version:
+        description: "Release version"
+        required: true
+        type: string
+
+jobs:
+  create_release_tag:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: create branch
+        run: |
+          set -x -e
+
+          branch_name="${{ github.ref_name }}"
+          echo "branch name: $branch_name"
+
+          if [[ $branch_name != release/promptflow/* ]]; then
+            echo "not a release branch for promptflow, exiting..."
+            exit 1
+          fi
+
+          release_version=${branch_name#release/promptflow/}
+          echo "release version: $release_version"
+
+          echo "replacing version in src/promptflow/promptflow/_version.py..."
+          sed -i "s/0.0.1/$release_version/g" src/promptflow/promptflow/_version.py
+          echo "replaced src/promptflow/promptflow/_version.py:"
+          cat src/promptflow/promptflow/_version.py
+
+          if [[ $(git diff --name-only) == *"src/promptflow/promptflow/_version.py"* ]]; then
+            git add src/promptflow/promptflow/_version.py
+            git config --global user.name 'promptflow release'
+            git config --global user.email 'aml-pt-eng@microsoft.com'
+            git commit -m "update version in _version.py for promptflow"
+            git push --set-upstream origin $branch_name
+          fi
+
+          git tag promptflow_$release_version
+          git push origin --tags
+
+          # write environment variable to `GITHUB_ENV` to pass values between steps
+          # https://docs.github.com/en/github-ae@latest/actions/learn-github-actions/variables#passing-values-between-steps-and-jobs-in-a-workflow
+          echo "release_version=$release_version" >> "$GITHUB_ENV"
+          echo "release_tag=$release_tag" >> "$GITHUB_ENV"
+
+      - name: create asset
+        run: |
+          cd src
+          tar -czvf promptflow-$release_version.tar.gz promptflow
+
+      - name: create release note
+        run: |
+          cp ./scripts/release/promptflow-release-note.md ./src/promptflow/release_note.md
+          sed -i "s/{{VERSION}}/$release_version/g" ./src/promptflow/release_note.md
+          cat ./src/promptflow/release_note.md
+
+      - name: create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: promptflow_${{ inputs.release_version }}
+          release_name: promptflow ${{ inputs.release_version }}
+          body_path: ./src/promptflow/release_note.md
+          draft: false
+          prerelease: false
+
+      - name: upload asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./src/promptflow-${{ inputs.release_version }}.tar.gz
+          asset_name: promptflow-${{ inputs.release_version }}.tar.gz
+          asset_content_type: application/gzip

.github/workflows/flake8.yml

@@ -0,0 +1,28 @@
+name: Flake8 Lint
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  flake8:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: install dependencies
+        run: pip install flake8
+
+      - name: run flake8
+        run: flake8

.github/workflows/flowdag_schema_check.yml

@@ -0,0 +1,35 @@
+name: examples_flowdag_schema_check
+
+on:
+  pull_request:
+    paths:
+      - examples/**
+      - .github/workflows/flowdag_schema_check.yml
+      - scripts/readme/schema_checker.py
+env:
+  IS_IN_CI_PIPELINE: "true"
+jobs:
+  examples_flowdag_schema_check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: env | sort >> $GITHUB_OUTPUT
+      - name: Python Setup - ubuntu-latest - Python Version 3.9
+        uses: "./.github/actions/step_create_python_environment"
+        with:
+          pythonVersion: 3.9
+      - run: |
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+      - name: Summarize check status
+        id: summarize_check_status
+        working-directory: ${{ github.workspace }}
+        shell: pwsh
+        env:
+          PYTHONPATH: ${{ github.workspace }}/src/promptflow
+        run: |
+          cd ${{ github.workspace }}/src
+          pip install ./promptflow[azure]
+          pip install ./promptflow-tools
+          python ${{ github.workspace }}/scripts/readme/schema_checker.py

.github/workflows/labeler.yml

@@ -0,0 +1,24 @@
+name: "Pull Request Labeler"
+on:
+- pull_request_target
+
+jobs:
+  triage:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v4
+
+    - name: apply a label to pull request from fork
+      if: ${{ github.event.pull_request.head.repo.full_name != 'microsoft/promptflow' }}
+      uses: actions/github-script@v7
+      with:
+        script: |
+          github.rest.issues.addLabels({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            labels: ['external']
+          })

.github/workflows/promptflow-core-test.yml

@@ -0,0 +1,167 @@
+name: promptflow-core-test [Pure]
+
+on:
+  schedule:
+    - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
+  pull_request:
+    paths:
+      - src/promptflow-tracing/**
+      - src/promptflow-core/**
+      - .github/workflows/promptflow-core-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  core_test:
+    environment:
+      internal
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: |
+          poetry install -E executor-service --with ci,test
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: set test mode
+        run: |
+          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - name: Azure login (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: generate live test resources (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: generate live test resources (pull_request workflow)
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+        run: |
+          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
+      - name: run core tests
+        run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/core/*.xml
+            ${{ env.WORKING_DIRECTORY }}/core/htmlcov/
+
+  azureml_serving_test:
+    environment:
+      internal
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: |
+          poetry install -E azureml-serving --with ci,test
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: set test mode
+        run: |
+          echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - name: Azure login (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: generate live test resources (non pull_request workflow)
+        if: github.event_name != 'pull_request'
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: generate live test resources (pull_request workflow)
+        if: github.event_name == 'pull_request'
+        working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+        run: |
+          cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
+      - name: run azureml-serving tests
+        run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/azureml-serving/*.xml
+            ${{ env.WORKING_DIRECTORY }}/azureml-serving/htmlcov/
+
+  report:
+    needs: [core_test, azureml_serving_test]
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-core test result
+          comment_title: promptflow-core test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+#     TODO: Enable coverage check after core test fully setup
+#      - uses: irongut/CodeCoverageSummary@v1.3.0
+#        with:
+#          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
+#          badge: true
+#          fail_below_min: true
+#          format: markdown
+#          hide_complexity: true
+#          output: both
+#          thresholds: 40 60

.github/workflows/promptflow-evals-e2e-test-azure.yml

@@ -0,0 +1,105 @@
+name: promptflow-evals-e2e-test-azure
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-e2e-test-azure.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        # Always run in replay mode for now until we figure out the test resource to run live mode
+        run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
+        #run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow packages in editable mode
+        run: |
+          poetry run pip install -e ../promptflow
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
+          poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
+          poetry run pip install -e ../promptflow-evals
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
+          tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
+          subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
+      - name: run e2e tests
+        id: run_e2e_tests_azure
+        run: |
+          poetry run pytest -m azuretest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity e2e_tests_azure --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-evals test result
+          comment_title: promptflow-evals test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 80

.github/workflows/promptflow-evals-e2e-test-local.yml

@@ -0,0 +1,107 @@
+name: promptflow-evals-e2e-test-local
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-e2e-test-local.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        # Always run in replay mode for now until we figure out the test resource to run live mode
+        run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
+        #run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow packages in editable mode
+        run: |
+          poetry run pip install -e ../promptflow
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
+          poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-evals
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
+          tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
+          subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
+      - name: check azure is not installed
+        run: poetry run pytest ../../scripts/code_qa/assert_local_install.py
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run e2e tests
+        id: run_e2e_tests_local
+        run: |
+          poetry run pytest -m localtest tests/evals/e2etests --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity e2e_tests_local --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-evals test result
+          comment_title: promptflow-evals test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 80

.github/workflows/promptflow-evals-installation-test.yml

@@ -0,0 +1,61 @@
+name: promptflow-evals-installation-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-installation-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+  PROMPT_FLOW_TEST_MODE: "live"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-evals
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_evals-*.whl
+
+  test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-evals
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install virtualenv
+        run: python -m pip install virtualenv
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-evals from wheel
+        id: install_promptflow_no_extras
+        run: |
+          bash ../../scripts/code_qa/calculate_install_time.sh -r ${{ github.run_id }} -w ${{ github.workflow }} -a ${{ github.action }} -b ${{ github.ref }} -l "300"
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-evals from wheel
+        id: install_promptflow_with_extras
+        run: |
+          bash ../../scripts/code_qa/calculate_install_time.sh -r ${{ github.run_id }} -w ${{ github.workflow }} -a ${{ github.action }} -b ${{ github.ref }} -e "[azure]" -l "300"
+        working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/promptflow-evals-performance-test.yml

@@ -0,0 +1,71 @@
+name: promptflow-evals-performance-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-performance-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+  PROMPT_FLOW_TEST_MODE: "live"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-evals from wheel
+        id: install_promptflow
+        run: |
+          # Estimate the installation time.
+          poetry run pip install -e ../promptflow
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
+          poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
+          poetry run pip install -e ../promptflow-evals
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.PF_EVALS_SP_CLIENT_ID }}
+          tenant-id: ${{ secrets.PF_EVALS_SP_TENANT_ID }}
+          subscription-id: ${{ secrets.PF_EVALS_SP_SUBSCRIPTION_ID }}
+      - name: run performance tests
+        id: performance_tests
+        run: |
+          # Estimate the run time for evaluator.
+          poetry run pytest -m performance_test --junit-xml=test-results.xml
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity evaluator_live_tests_run_time_s --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/promptflow-evals-unit-test.yml

@@ -0,0 +1,88 @@
+name: promptflow-evals-unit-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-unit-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow packages in editable mode
+        run: |
+          poetry run pip install -e ../promptflow
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
+          poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
+          poetry run pip install -e ../promptflow-evals
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run unit tests
+        id: run_unit_tests
+        run: |
+            poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --cov-fail-under=58
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-evals test result
+          comment_title: promptflow-evals test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 60

.github/workflows/promptflow-executor-e2e-test.yml

@@ -0,0 +1,173 @@
+name: promptflow-executor-e2e-test
+on:
+  schedule:
+    - cron: "40 20 * * *" # Every day starting at 4:40 BJT
+  workflow_dispatch:
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+  PYTHONPATH: ${{ github.workspace }}/src/promptflow
+  IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  authorize:
+    environment:
+      # forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
+      ${{ github.event_name == 'pull_request_target' &&
+      github.event.pull_request.head.repo.full_name != github.repository &&
+      'external' || 'internal' }}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+  build:
+    needs: authorize
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ubuntu-latest - Python Version 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Build wheel
+      uses: "./.github/actions/step_sdk_setup"
+      with:
+        setupType: promptflow_with_extra
+        scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Upload Wheel
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+
+  executor_e2e_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    environment:
+      internal
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Set test mode
+      run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Python Setup - ${{ matrix.os }} - Python Version 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Install wheel
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        Set-PSDebug -Trace 1
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+        tenant-id: ${{secrets.AZURE_TENANT_ID}}
+        client-id: ${{secrets.AZURE_CLIENT_ID}}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v1
+      id: cpu-cores
+    - name: Run Coverage Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        gci env:* | sort-object name
+        az account show
+        pip install langchain-community tenacity<8.4.0
+        # numexpr is required by langchain in e2e tests.
+        pip install numexpr
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ env.testWorkingDirectory }}/promptflow `
+          -t ${{ env.testWorkingDirectory }}/tests/executor/e2etests `
+          -l eastus `
+          -m "all" `
+          -n ${{ steps.cpu-cores.outputs.count }}`
+          --coverage-config ${{ env.testWorkingDirectory }}/tests/executor/.coveragerc `
+          --disable-cov-branch
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python 3.9) (OS ${{ matrix.os }})
+        path: |
+          ${{ github.workspace }}/*.xml
+          ${{ github.workspace }}/htmlcov/
+  publish-test-results:
+    name: "Publish Tests Results"
+    needs: executor_e2e_tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-executor-e2e-test.yml
+        testResultTitle: Executor E2E Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 70
+        context: test/executor_e2e

.github/workflows/promptflow-executor-unit-test.yml

@@ -0,0 +1,175 @@
+name: promptflow-executor-unit-test
+on:
+  schedule:
+    - cron: "40 19 * * *" # Every day starting at 3:40 BJT
+  workflow_dispatch:
+permissions:
+  id-token: write
+  contents: read
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+  PYTHONPATH: ${{ github.workspace }}/src/promptflow
+  IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+jobs:
+  authorize:
+    environment:
+      # forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
+      ${{ github.event_name == 'pull_request_target' &&
+      github.event.pull_request.head.repo.full_name != github.repository &&
+      'external' || 'internal' }}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+  build:
+    needs: authorize
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ubuntu-latest - Python Version 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Build wheel
+      uses: "./.github/actions/step_sdk_setup"
+      with:
+        setupType: promptflow_with_extra
+        scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Upload Wheel
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+  executor_unit_tests:
+    needs: build
+    environment:
+      internal
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Set test mode
+      run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - name: Python Setup - ${{ matrix.os }} - Python Version 3.9
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: 3.9
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: Install wheel
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        Set-PSDebug -Trace 1
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+        pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+        tenant-id: ${{secrets.AZURE_TENANT_ID}}
+        client-id: ${{secrets.AZURE_CLIENT_ID}}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v1
+      id: cpu-cores
+    - name: Run Coverage Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        gci env:* | sort-object name
+        az account show
+        pip install langchain-community tenacity<8.4.0
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ env.testWorkingDirectory }}/promptflow `
+          -t ${{ env.testWorkingDirectory }}/tests/executor/unittests `
+          -l eastus `
+          -m "all" `
+          -n ${{ steps.cpu-cores.outputs.count }} `
+          --coverage-config ${{ env.testWorkingDirectory }}/tests/executor/.coveragerc `
+          --disable-cov-branch
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python 3.9) (OS ${{ matrix.os }})
+        path: |
+          ${{ github.workspace }}/*.xml
+          ${{ github.workspace }}/htmlcov/
+  publish-test-results:
+    name: "Publish Tests Results"
+    needs: executor_unit_tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-executor-unit-test.yml
+        testResultTitle: Executor Unit Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 50
+        token: ${{ secrets.GITHUB_TOKEN }}
+        context: test/executor_unit

.github/workflows/promptflow-global-config-test.yml

@@ -0,0 +1,126 @@
+name: promptflow-global-config-test
+on:
+  schedule:
+    - cron: "40 18 * * *" # Every day starting at 2:40 BJT
+  workflow_dispatch:
+permissions:
+  id-token: write
+  contents: read
+env:
+  IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+  AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+jobs:
+  authorize:
+    environment:
+      # forked prs from pull_request_target will be run in external environment, domain prs will be run in internal environment
+      ${{ github.event_name == 'pull_request_target' &&
+      github.event.pull_request.head.repo.full_name != github.repository &&
+      'external' || 'internal' }}
+    runs-on: ubuntu-latest
+    steps:
+      - run: true
+  sdk_cli_global_config_tests:
+    needs: authorize
+    environment:
+      internal
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+        fetch-depth: 0
+    - name: merge main to current branch
+      uses: "./.github/actions/step_merge_main"
+    - name: Display and Set Environment Variables
+      run: |
+        export pyVersion="3.9"
+        env | sort >> $GITHUB_OUTPUT
+      id: display_env
+      shell: bash -el {0}
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ steps.display_env.outputs.pyVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+      run: |
+        set -xe
+        poetry install --only test
+        poetry run pip install ${{ env.TRACING_DIRECTORY }}
+        poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving]
+        poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow]
+        poetry run pip install -e ${{ env.AZURE_DIRECTORY }}
+
+        echo "Need to install promptflow to avoid tool dependency issue"
+        poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }}
+        poetry run pip install ${{ env.TOOL_DIRECTORY }}
+        poetry run pip install -e ${{ env.RECORD_DIRECTORY }}
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-azure
+        poetry run pip show promptflow-tools
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+        tenant-id: ${{secrets.AZURE_TENANT_ID}}
+        client-id: ${{secrets.AZURE_CLIENT_ID}}
+    - name: Install Azure Login items
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+      run: |
+        pip install azure-identity
+        pip install azure-keyvault
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+    - name: run devkit tests
+      run: |
+        poetry run pytest ./tests/sdk_cli_global_config_test --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" --tb=short
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }})
+        path: |
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
+  publish-test-results-global-config-test:
+    needs: sdk_cli_global_config_tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-global-config-test.yml
+        testResultTitle: SDK CLI Global Config Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 0
+        context: test/sdk_cli

.github/workflows/promptflow-import-linter.yml

@@ -0,0 +1,66 @@
+name: promptflow-import-linter
+
+on:
+  pull_request:
+    paths:
+      - src/promptflow-tracing/**
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow-azure/**
+      - .github/workflows/promptflow-import-linter.yml
+  workflow_dispatch:
+
+env:
+  WORKING_DIRECTORY: ${{ github.workspace }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - uses: snok/install-poetry@v1
+    - name: Install all packages
+      run: |
+        touch src/promptflow-tracing/promptflow/__init__.py
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing --with dev
+        touch src/promptflow-core/promptflow/__init__.py
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core --with dev
+        touch src/promptflow-devkit/promptflow/__init__.py
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit --with dev
+        touch src/promptflow-azure/promptflow/__init__.py
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure --with dev
+        touch src/promptflow-evals/promptflow/__init__.py
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals --with dev
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: import lint
+      run: |
+        echo "=== Running import lint in promptflow-tracing ==="
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing run lint-imports
+        echo "=== Running import lint in promptflow-core ==="
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core run lint-imports
+        echo "=== Running import lint in promptflow-devkit ==="
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit run lint-imports
+        echo "=== Running import lint in promptflow-azure ==="
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure run lint-imports
+        echo "=== Running import lint in promptflow-evals ==="
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals run lint-imports
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: import lint testing private imports from global
+      working-directory: ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure
+      run: |
+        set -xe
+        rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing/promptflow/__init__.py
+        rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-core/promptflow/__init__.py
+        rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit/promptflow/__init__.py
+        rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure/promptflow/__init__.py
+        rm ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals/promptflow/__init__.py
+        echo "=== Add more import linter when facing more import errors ==="
+
+        echo "=== promptflow-azure full lints ==="
+        poetry run pip install langchain
+        poetry run pip install "tenacity<8.4.0"
+        poetry run python ${{ github.workspace }}/scripts/import_linter/import_linter.py

.github/workflows/promptflow-parallel-e2e-test.yml

@@ -0,0 +1,87 @@
+name: promptflow-parallel-e2e-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow/**
+      - src/promptflow-core/**
+      - src/promptflow-tracing/**
+      - src/promptflow-parallel/**
+      - .github/workflows/promptflow-parallel-e2e-test.yml
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-parallel
+
+jobs:
+  parallel-e2e-test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    environment:
+      internal
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        # Always run in replay mode for now until we figure out the test resource to run live mode
+        run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
+        #run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install promptflow packages in editable mode
+        run: |
+          set -xe
+          poetry install --with ci,test
+
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run e2e tests
+        run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  parallel-e2e-test-report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-parallel-e2e-test.yml
+        testResultTitle: Parallel E2E Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 40
+        context: test/parallel-e2e

.github/workflows/promptflow-parallel-unit-test.yml

@@ -0,0 +1,87 @@
+name: promptflow-parallel-unit-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow/**
+      - src/promptflow-core/**
+      - src/promptflow-tracing/**
+      - src/promptflow-parallel/**
+      - .github/workflows/promptflow-parallel-unit-test.yml
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-parallel
+
+jobs:
+  parallel-unit-test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    environment:
+      internal
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        # Always run in replay mode for now until we figure out the test resource to run live mode
+        run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
+        #run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install promptflow packages in editable mode
+        run: |
+          set -xe
+          poetry install --with ci,test
+
+          poetry run pip show promptflow-tracing
+          poetry run pip show promptflow-core
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run unit tests
+        run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results (Python ${{ matrix.python-version }}) (OS ${{ matrix.os }})
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  parallel-unit-test-report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-parallel-unit-test.yml
+        testResultTitle: Parallel Unit Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 40
+        context: test/parallel-unit

.github/workflows/promptflow-release-testing-matrix.yml

@@ -0,0 +1,468 @@
+name: promptflow-release-testing-matrix
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      # can leave empty when trigger manually
+      # GitHub Actions API for trigger does not return workflow run id
+      # there we reference below Stack Overflow solution:
+      # https://stackoverflow.com/a/69500478
+      # which adds an identifier in workflow run jobs and can be used for filter
+      id:
+        description: Identifier for the workflow run
+        required: false
+        type: string
+env:
+  IS_IN_CI_PIPELINE: "true"
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  AZURE_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  DEVKIT_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  id:
+    runs-on: ubuntu-latest
+    steps:
+      - name: workflow run id - ${{ inputs.id }}
+        run: |
+          echo "workflow run id: ${{ inputs.id }}"
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.9"
+    - uses: snok/install-poetry@v1
+    - working-directory: ${{ env.TRACING_DIRECTORY }}
+      run: poetry build -f wheel
+    - working-directory: ${{ env.CORE_DIRECTORY }}
+      run: poetry build -f wheel
+    - working-directory: ${{ env.DEVKIT_DIRECTORY }}
+      run: poetry build -f wheel
+    - working-directory: ${{ env.AZURE_DIRECTORY }}
+      run: poetry build -f wheel
+    - working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+      run: |
+        pip install -r ./dev_requirements.txt
+        python ./setup.py bdist_wheel
+    - working-directory: ${{ env.TOOL_DIRECTORY }}
+      run: python ./setup.py bdist_wheel
+
+    - name: Upload Wheel
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheel
+        path: |
+          ${{ github.workspace }}/src/promptflow/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tracing/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-core/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-devkit/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-azure/dist/*.whl
+          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+
+  promptflow_tracing_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run:
+        env | sort >> $GITHUB_OUTPUT
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.pythonVersion }}
+    - uses: snok/install-poetry@v1
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: ${{ env.WORKING_DIRECTORY }}/artifacts
+    - name: install promptflow-tracing from wheel
+      # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+      run: poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: install test dependency group
+      run: poetry install --only test
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: install recording
+      run: poetry install
+      working-directory: ${{ env.RECORD_DIRECTORY }}
+    - name: generate end-to-end test config from secret
+      run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: run e2e tests
+      run: poetry run pytest -m e2etest --tb=short
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: upload coverage report
+      uses: actions/upload-artifact@v4
+      with:
+        name: promptflow_tracing_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
+        path: |
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+
+
+  promptflow_core_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    environment:
+      internal
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: ${{ env.WORKING_DIRECTORY }}/artifacts
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Generate Configs
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: install promptflow-core from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: |
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0])")
+          poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run core tests
+        run: poetry run pytest ./tests/core --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: promptflow_core_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+
+  promptflow_core_azureml_serving_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: ${{ env.WORKING_DIRECTORY }}/artifacts
+      - name: install promptflow-core from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: |
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run azureml-serving tests
+        run: poetry run pytest ./tests/azureml-serving --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: promptflow_core_azureml_serving_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+
+
+  promptflow_devkit_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    environment:
+      internal
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: ${{ env.WORKING_DIRECTORY }}/artifacts
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Generate Configs
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: install promptflow-devkit from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: |
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run devkit tests
+        run: poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -n auto -m "unittest or e2etest" --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: promptflow_devkit_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+
+
+  promptflow_azure_tests:
+    needs: build
+    environment:
+      internal
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    env:
+      PROMPT_FLOW_TEST_MODE: "live"
+      WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
+      PROMPT_FLOW_WORKSPACE_NAME: "promptflow-eastus"
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.pythonVersion }}
+      - uses: snok/install-poetry@v1
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: ${{ env.WORKING_DIRECTORY }}/artifacts
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Generate Configs
+        uses: "./.github/actions/step_generate_configs"
+        with:
+          targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+      - name: install promptflow-azure from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: |
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tracing-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_core-*.whl', recursive=True)[0]+'[azureml-serving]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_devkit-*.whl', recursive=True)[0]+'[executable]')")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_azure-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow-*.whl', recursive=True)[0])")
+          poetry run pip install $(python -c "import glob; print(glob.glob('**/promptflow_tools-*.whl', recursive=True)[0])")
+          poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run azure tests
+        run: poetry run pytest ./tests/sdk_cli_azure_test -n auto -m "unittest or e2etest" --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: promptflow_azure_tests report-${{ matrix.os }}-py${{ matrix.pythonVersion }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+
+
+  promptflow_executor_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    environment:
+      internal
+    env:
+      testWorkingDirectory: src/promptflow
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Display and Set Environment Variables
+      run:
+        env | sort >> $GITHUB_OUTPUT
+      shell: bash -el {0}
+    - name: Python Env Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
+      uses: "./.github/actions/step_create_python_environment"
+      with:
+        pythonVersion: ${{ matrix.pythonVersion }}
+    - name: Download Artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheel
+        path: artifacts
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
+    - name: Azure Login
+      uses: azure/login@v1
+      with:
+        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+        tenant-id: ${{secrets.AZURE_TENANT_ID}}
+        client-id: ${{secrets.AZURE_CLIENT_ID}}
+    - name: Generate Configs
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.testWorkingDirectory }}
+    - name: Install pf
+      shell: pwsh
+      working-directory: artifacts
+      run: |
+        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip uninstall -y promptflow-core promptflow-devkit promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executor-service]"}}
+        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
+        pip freeze
+    - name: Run Executor Unit Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        pip install langchain
+        pip install numexpr
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ github.workspace }}/src/promptflow/promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/unittests `
+          -l eastus `
+          -m "all" `
+          -o "${{ github.workspace }}/test-results-executor-unit.xml"
+    - name: Run Executor E2E Test
+      shell: pwsh
+      working-directory: ${{ github.workspace }}
+      run: |
+        pip install langchain langchain_community
+        pip install numexpr
+        python scripts/building/run_coverage_tests.py `
+          -p ${{ github.workspace }}/src/promptflow/promptflow `
+          -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests `
+          -l eastus `
+          -m "all" `
+          -o "${{ github.workspace }}/test-results-executor-e2e.xml"
+    - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: promptflow_executor_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: ${{ github.workspace }}/*.xml
+
+
+  publish-test-results:
+    name: "Publish Tests Results"
+    needs: [ promptflow_devkit_tests, promptflow_azure_tests, promptflow_executor_tests, promptflow_core_tests, promptflow_core_azureml_serving_tests ]
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "artifacts/**/test-*.xml"

.github/workflows/promptflow-sdk-cli-test.yml

@@ -0,0 +1,124 @@
+name: promptflow-sdk-cli-test
+on:
+  schedule:
+    - cron: "40 18 * * *" # Every day starting at 2:40 BJT
+  pull_request:
+    paths:
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
+      - src/promptflow/**
+      - src/promptflow-tracing/**
+      - scripts/building/**
+      - .github/workflows/promptflow-sdk-cli-test.yml
+      - src/promptflow-recording/**
+  workflow_dispatch:
+    inputs:
+      filepath:
+        description: file or paths you want to trigger a test
+        required: true
+        default: "./tests/sdk_cli_test ./tests/sdk_pfs_test"
+        type: string
+permissions:
+  id-token: write
+  contents: read
+env:
+  IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit
+  PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow
+  TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools
+jobs:
+  sdk_cli_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pythonVersion: ['3.9', '3.10', '3.11']
+    environment:
+      internal
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: set test mode
+      run: |
+        echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+        echo "FILE_PATHS=$(if [[ "${{ inputs.filepath }}" == "" ]]; then echo "./tests/sdk_cli_test ./tests/sdk_pfs_test"; else echo ${{ inputs.filepath }}; fi)" >> $GITHUB_ENV
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.pythonVersion }}
+    - uses: snok/install-poetry@v1
+    - name: install test dependency group
+      run: |
+        set -xe
+        poetry install -E pyarrow --with ci,test
+
+        poetry run pip show promptflow-tracing
+        poetry run pip show promptflow-core
+        poetry run pip show promptflow-devkit
+        poetry run pip show promptflow-tools
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: Azure login (non pull_request workflow)
+      if: github.event_name != 'pull_request'
+      uses: azure/login@v1
+      with:
+        subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+        tenant-id: ${{secrets.AZURE_TENANT_ID}}
+        client-id: ${{secrets.AZURE_CLIENT_ID}}
+    - name: generate live test resources (non pull_request workflow)
+      if: github.event_name != 'pull_request'
+      uses: "./.github/actions/step_generate_configs"
+      with:
+        targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }}
+    - name: generate live test resources (pull_request workflow)
+      if: github.event_name == 'pull_request'
+      working-directory: ${{ env.PROMPTFLOW_DIRECTORY }}
+      run: |
+        cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json
+    - name: run devkit tests
+      run: |
+        poetry run pytest ${{ env.FILE_PATHS }} --cov=promptflow --cov-config=pyproject.toml \
+        --cov-report=term --cov-report=html --cov-report=xml -n auto -m "(unittest or e2etest) and not csharp" \
+        --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: Upload Test Results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        path: |
+          ${{ env.WORKING_DIRECTORY }}/*.xml
+          ${{ env.WORKING_DIRECTORY }}/htmlcov/
+          ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json
+    - run: poetry install -E executable
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - name: run devkit executable tests
+      run: |
+        poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py --tb=short
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+
+
+  publish-test-results-sdk-cli-test:
+    needs: sdk_cli_tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    if: always()
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: Publish Test Results
+      uses: "./.github/actions/step_publish_test_results"
+      with:
+        testActionFileName: promptflow-sdk-cli-test.yml
+        testResultTitle: SDK CLI Test Result
+        osVersion: ubuntu-latest
+        pythonVersion: 3.9
+        coverageThreshold: 40
+        context: test/sdk_cli

.github/workflows/promptflow-tracing-e2e-test.yml

@@ -0,0 +1,109 @@
+name: promptflow-tracing-e2e-test
+
+on:
+  schedule:
+    - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
+  pull_request:
+    paths:
+      - src/promptflow-tracing/**
+      - .github/workflows/promptflow-tracing-e2e-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-tracing
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
+
+  tracing-e2e-test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-tracing
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-tracing from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: poetry run pip install $(python -c "import glob; print(glob.glob('promptflow_tracing-*.whl')[0])")
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ env.RECORD_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run e2e tests
+        run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: tracing-e2e-test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-tracing test result
+          comment_title: promptflow-tracing test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
+          badge: true
+          fail_below_min: true
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 80

.github/workflows/promptflow-tracing-unit-test.yml

@@ -0,0 +1,100 @@
+name: promptflow-tracing-unit-test
+
+on:
+  schedule:
+    - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
+  pull_request:
+    paths:
+      - src/promptflow-tracing/**
+      - .github/workflows/promptflow-tracing-unit-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-tracing
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
+
+  tracing-unit-test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-tracing
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-tracing from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: poetry run pip install $(python -c "import glob; print(glob.glob('promptflow_tracing-*.whl')[0])")
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ env.RECORD_DIRECTORY }}
+      - name: run unit tests
+        run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --tb=short
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: tracing-unit-test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-tracing test result
+          comment_title: promptflow-tracing test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
+          badge: true
+          fail_below_min: true
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 60

.github/workflows/publish_doc.yml

@@ -0,0 +1,87 @@
+name: Publish Promptflow Doc
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - preview/docs
+    paths:
+      - 'README.md'
+      - 'docs/**'
+      - 'examples/**.ipynb'
+      - 'scripts/docs/**'
+      - '.github/workflows/publish_doc.yml'
+      - 'src/promptflow-tracing/promptflow/**'
+      - 'src/promptflow-core/promptflow/**'
+      - 'src/promptflow-devkit/promptflow/**'
+      - 'src/promptflow-azure/promptflow/**'
+      - 'src/promptflow-rag/promptflow/**'
+      - 'src/promptflow-evals/promptflow/**'
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+
+# Allow one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+env:
+  packageSetupType: promptflow_with_extra
+  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
+
+jobs:
+  build:
+    runs-on: windows-latest
+    name: Build
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag promptflow-evals
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
+          pip install -e ${{ github.workspace }}/src/promptflow-evals
+          pip freeze
+
+      - name: Build Doc
+        shell: powershell
+        working-directory: scripts/docs/
+        run: |-
+          pip install langchain
+          ./doc_generation.ps1 -WithReferenceDoc:$true
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          # Upload entire repository
+          path: scripts/docs/_build
+      
+  deploy:
+    runs-on: ubuntu-latest
+    name: Deploy
+    needs: build
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/pylint.yml

@@ -0,0 +1,32 @@
+name: Pylint
+
+on:
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+
+jobs:
+  run_pylint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: snok/install-poetry@v1
+      - name: install pylint and azure-pylint-guidelines-checker
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: |
+          set -xe
+          poetry install -C src/promptflow-evals --with dev
+          poetry show -C src/promptflow-evals
+      - name: run pylint
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+        run: |
+          cd src/promptflow-evals
+          poetry run pylint promptflow/evals --rcfile=../../pylintrc

.github/workflows/samples_configuration.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_configuration
+on:
+  schedule:
+    - cron: "40 22 * * *" # Every day starting at 6:40 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/configuration.ipynb, .github/workflows/samples_configuration.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_configuration:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples
+        run: |
+          papermill -k python configuration.ipynb configuration.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples

.github/workflows/samples_connections.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_connections
+on:
+  schedule:
+    - cron: "40 20 * * *" # Every day starting at 4:40 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/connections/**, examples/*requirements.txt, .github/workflows/samples_connections.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_connections:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/connections
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/connections
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/connections/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/connections/README.md -o examples/connections
+      - name: Cat script
+        working-directory: examples/connections
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/connections
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/connections
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/connections
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/connections/bash_script.sh

.github/workflows/samples_connections_connection.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_connections_connection
+on:
+  schedule:
+    - cron: "13 22 * * *" # Every day starting at 6:13 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/connections/**, examples/*requirements.txt, .github/workflows/samples_connections_connection.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_connections_connection:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/connections
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/connections
+        run: |
+          papermill -k python connection.ipynb connection.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/connections

.github/workflows/samples_flex_flows_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_basic
+on:
+  schedule:
+    - cron: "30 20 * * *" # Every day starting at 4:30 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/basic/README.md -o examples/flex-flows/basic
+      - name: Cat script
+        working-directory: examples/flex-flows/basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/basic/bash_script.sh

.github/workflows/samples_flex_flows_chat_async_stream.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_chat_async_stream
+on:
+  schedule:
+    - cron: "59 20 * * *" # Every day starting at 4:59 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-async-stream/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_async_stream.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_chat_async_stream:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/chat-async-stream/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-async-stream/README.md -o examples/flex-flows/chat-async-stream
+      - name: Cat script
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-async-stream/bash_script.sh

.github/workflows/samples_flex_flows_chat_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_chat_basic
+on:
+  schedule:
+    - cron: "9 20 * * *" # Every day starting at 4:9 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_chat_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/chat-basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-basic/README.md -o examples/flex-flows/chat-basic
+      - name: Cat script
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-basic/bash_script.sh

.github/workflows/samples_flex_flows_chat_minimal.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_chat_minimal
+on:
+  schedule:
+    - cron: "29 22 * * *" # Every day starting at 6:29 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-minimal/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_minimal.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_chat_minimal:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/chat-minimal/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-minimal/README.md -o examples/flex-flows/chat-minimal
+      - name: Cat script
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/chat-minimal
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-minimal/bash_script.sh

.github/workflows/samples_flex_flows_chat_stream.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_chat_stream
+on:
+  schedule:
+    - cron: "7 19 * * *" # Every day starting at 3:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-stream/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_stream.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_chat_stream:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/chat-stream/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-stream/README.md -o examples/flex-flows/chat-stream
+      - name: Cat script
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-stream/bash_script.sh

.github/workflows/samples_flex_flows_chat_with_functions.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_chat_with_functions
+on:
+  schedule:
+    - cron: "51 21 * * *" # Every day starting at 5:51 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-with-functions/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_chat_with_functions.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_chat_with_functions:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/chat-with-functions/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/chat-with-functions/README.md -o examples/flex-flows/chat-with-functions
+      - name: Cat script
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/chat-with-functions
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-with-functions/bash_script.sh

.github/workflows/samples_flex_flows_eval_checklist.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_eval_checklist
+on:
+  schedule:
+    - cron: "56 22 * * *" # Every day starting at 6:56 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-checklist/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_checklist.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_eval_checklist:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/eval-checklist/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-checklist/README.md -o examples/flex-flows/eval-checklist
+      - name: Cat script
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/eval-checklist
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-checklist/bash_script.sh

.github/workflows/samples_flex_flows_eval_code_quality.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_eval_code_quality
+on:
+  schedule:
+    - cron: "6 22 * * *" # Every day starting at 6:6 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-code-quality/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_code_quality.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_eval_code_quality:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/eval-code-quality/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-code-quality/README.md -o examples/flex-flows/eval-code-quality
+      - name: Cat script
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/eval-code-quality
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-code-quality/bash_script.sh

.github/workflows/samples_flex_flows_eval_criteria_with_langchain.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_eval_criteria_with_langchain
+on:
+  schedule:
+    - cron: "21 20 * * *" # Every day starting at 4:21 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-criteria-with-langchain/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_criteria_with_langchain.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flex_flows_eval_criteria_with_langchain:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flex-flows/eval-criteria-with-langchain/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-criteria-with-langchain/README.md -o examples/flex-flows/eval-criteria-with-langchain
+      - name: Cat script
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-criteria-with-langchain/bash_script.sh

.github/workflows/samples_flexflows_basic_flexflowquickstart.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_basic_flexflowquickstart
+on:
+  schedule:
+    - cron: "55 20 * * *" # Every day starting at 4:55 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_basic_flexflowquickstart.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_basic_flexflowquickstart:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/basic
+        run: |
+          papermill -k python flex-flow-quickstart.ipynb flex-flow-quickstart.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/basic

.github/workflows/samples_flexflows_basic_flexflowquickstartazure.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_basic_flexflowquickstartazure
+on:
+  schedule:
+    - cron: "10 22 * * *" # Every day starting at 6:10 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_basic_flexflowquickstartazure.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_basic_flexflowquickstartazure:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/basic
+        run: |
+          papermill -k python flex-flow-quickstart-azure.ipynb flex-flow-quickstart-azure.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/basic

.github/workflows/samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow
+on:
+  schedule:
+    - cron: "11 20 * * *" # Every day starting at 4:11 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-async-stream/**, examples/*requirements.txt, .github/workflows/samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_chatasyncstream_chatstreamwithasyncflexflow:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/chat-async-stream
+        run: |
+          papermill -k python chat-stream-with-async-flex-flow.ipynb chat-stream-with-async-flex-flow.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-async-stream

.github/workflows/samples_flexflows_chatbasic_chatwithclassbasedflow.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_chatbasic_chatwithclassbasedflow
+on:
+  schedule:
+    - cron: "32 22 * * *" # Every day starting at 6:32 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_chatbasic_chatwithclassbasedflow.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_chatbasic_chatwithclassbasedflow:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          papermill -k python chat-with-class-based-flow.ipynb chat-with-class-based-flow.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-basic

.github/workflows/samples_flexflows_chatbasic_chatwithclassbasedflowazure.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_chatbasic_chatwithclassbasedflowazure
+on:
+  schedule:
+    - cron: "46 20 * * *" # Every day starting at 4:46 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_flexflows_chatbasic_chatwithclassbasedflowazure.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_chatbasic_chatwithclassbasedflowazure:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/chat-basic
+        run: |
+          papermill -k python chat-with-class-based-flow-azure.ipynb chat-with-class-based-flow-azure.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-basic

.github/workflows/samples_flexflows_chatstream_chatstreamwithflexflow.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_chatstream_chatstreamwithflexflow
+on:
+  schedule:
+    - cron: "28 21 * * *" # Every day starting at 5:28 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/chat-stream/**, examples/*requirements.txt, .github/workflows/samples_flexflows_chatstream_chatstreamwithflexflow.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_chatstream_chatstreamwithflexflow:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/chat-stream
+        run: |
+          papermill -k python chat-stream-with-flex-flow.ipynb chat-stream-with-flex-flow.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/chat-stream

.github/workflows/samples_flexflows_evalcriteriawithlangchain_langchaineval.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_evalcriteriawithlangchain_langchaineval
+on:
+  schedule:
+    - cron: "42 20 * * *" # Every day starting at 4:42 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-criteria-with-langchain/**, examples/*requirements.txt, .github/workflows/samples_flexflows_evalcriteriawithlangchain_langchaineval.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flexflows_evalcriteriawithlangchain_langchaineval:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          papermill -k python langchain-eval.ipynb langchain-eval.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-criteria-with-langchain

.github/workflows/samples_flows_chat_chat_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chat_basic
+on:
+  schedule:
+    - cron: "55 20 * * *" # Every day starting at 4:55 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chat_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chat_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/chat-basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/chat-basic/README.md -o examples/flows/chat/chat-basic
+      - name: Cat script
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/chat-basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-basic/bash_script.sh

.github/workflows/samples_flows_chat_chat_math_variant.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chat_math_variant
+on:
+  schedule:
+    - cron: "35 20 * * *" # Every day starting at 4:35 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-math-variant/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chat_math_variant.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chat_math_variant:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/chat-math-variant/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/chat-math-variant/README.md -o examples/flows/chat/chat-math-variant
+      - name: Cat script
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/chat-math-variant
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-math-variant/bash_script.sh

.github/workflows/samples_flows_chat_chat_with_image.yml

@@ -0,0 +1,121 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chat_with_image
+on:
+  schedule:
+    - cron: "16 19 * * *" # Every day starting at 3:16 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-with-image/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chat_with_image.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chat_with_image:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_GPT_4V_KEY }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          cp ../../../connections/azure_openai.yml ./azure_openai.yml
+          sed -i -e "s/<user-input>/$AOAI_API_KEY/g" -e "s/aoai-api-endpoint/$AOAI_API_ENDPOINT/g" azure_openai.yml
+      - name: Create AOAI Connection from ENV file
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          if [[ -e .env ]]; then
+            pf connection create --file .env --name aoai_gpt4v_connection
+          fi
+          if [[ -e azure_openai.yml ]]; then
+            pf connection create --file azure_openai.yml --name aoai_gpt4v_connection
+          fi
+          pf connection list
+      - name: Create run.yml
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/chat-with-image/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/chat-with-image/README.md -o examples/flows/chat/chat-with-image
+      - name: Cat script
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          export aoai_api_key=${{secrets.AOAI_GPT_4V_KEY }}
+          export aoai_api_endpoint=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          export aoai_api_key=${{secrets.AOAI_GPT_4V_KEY }}
+          export aoai_api_endpoint=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/chat-with-image
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-with-image/bash_script.sh

.github/workflows/samples_flows_chat_chat_with_pdf.yml

@@ -0,0 +1,133 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chat_with_pdf
+on:
+  schedule:
+    - cron: "4 22 * * *" # Every day starting at 6:4 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-with-pdf/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chat_with_pdf.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chat_with_pdf:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create AOAI Connection from ENV file
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          if [[ -e .env ]]; then
+            pf connection create --file .env --name chat_with_pdf_custom_connection
+          fi
+          if [[ -e azure_openai.yml ]]; then
+            pf connection create --file azure_openai.yml --name chat_with_pdf_custom_connection
+          fi
+          pf connection list
+      - name: Create run.yml
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/chat-with-pdf/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/chat-with-pdf/README.md -o examples/flows/chat/chat-with-pdf
+      - name: Cat script
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-with-pdf/bash_script.sh

.github/workflows/samples_flows_chat_chat_with_wikipedia.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chat_with_wikipedia
+on:
+  schedule:
+    - cron: "10 22 * * *" # Every day starting at 6:10 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-with-wikipedia/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chat_with_wikipedia.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chat_with_wikipedia:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/chat-with-wikipedia/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/chat-with-wikipedia/README.md -o examples/flows/chat/chat-with-wikipedia
+      - name: Cat script
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/chat-with-wikipedia
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-with-wikipedia/bash_script.sh

.github/workflows/samples_flows_chat_chatwithpdf_chatwithpdf.yml

@@ -0,0 +1,103 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chatwithpdf_chatwithpdf
+on:
+  schedule:
+    - cron: "12 21 * * *" # Every day starting at 5:12 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-with-pdf/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chatwithpdf_chatwithpdf.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chatwithpdf_chatwithpdf:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+          pip freeze
+          pf --version
+        shell: bash -el {0}
+      - name: Prepare sample requirements
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          pip install -r requirements.txt
+      - name: Create Chat With PDF Custom Connection
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+            pf connection create --file .env --name chat_with_pdf_custom_connection
+          fi
+      - name: Create AOAI Connection
+        working-directory: examples/connections
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          if [[ -e azure_openai.yml ]]; then
+            pf connection create --file azure_openai.yml --set api_key=$AOAI_API_KEY api_base=$AOAI_API_ENDPOINT
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          papermill -k python chat-with-pdf.ipynb chat-with-pdf.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-with-pdf

.github/workflows/samples_flows_chat_chatwithpdf_chatwithpdfazure.yml

@@ -0,0 +1,103 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_chatwithpdf_chatwithpdfazure
+on:
+  schedule:
+    - cron: "44 20 * * *" # Every day starting at 4:44 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/chat-with-pdf/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_chatwithpdf_chatwithpdfazure.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_chatwithpdf_chatwithpdfazure:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+          pip freeze
+          pf --version
+        shell: bash -el {0}
+      - name: Prepare sample requirements
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          pip install -r requirements.txt
+      - name: Create Chat With PDF Custom Connection
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+            pf connection create --file .env --name chat_with_pdf_custom_connection
+          fi
+      - name: Create AOAI Connection
+        working-directory: examples/connections
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          if [[ -e azure_openai.yml ]]; then
+            pf connection create --file azure_openai.yml --set api_key=$AOAI_API_KEY api_base=$AOAI_API_ENDPOINT
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          papermill -k python chat-with-pdf-azure.ipynb chat-with-pdf-azure.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/chat/chat-with-pdf

.github/workflows/samples_flows_chat_use_functions_with_chat_models.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_chat_use_functions_with_chat_models
+on:
+  schedule:
+    - cron: "7 21 * * *" # Every day starting at 5:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/chat/use_functions_with_chat_models/**, examples/*requirements.txt, .github/workflows/samples_flows_chat_use_functions_with_chat_models.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_chat_use_functions_with_chat_models:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/chat/use_functions_with_chat_models/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/chat/use_functions_with_chat_models/README.md -o examples/flows/chat/use_functions_with_chat_models
+      - name: Cat script
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/chat/use_functions_with_chat_models
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/chat/use_functions_with_chat_models/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_basic
+on:
+  schedule:
+    - cron: "2 19 * * *" # Every day starting at 3:2 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-basic/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-basic/README.md -o examples/flows/evaluation/eval-basic
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-basic/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_chat_math.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_chat_math
+on:
+  schedule:
+    - cron: "52 19 * * *" # Every day starting at 3:52 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-chat-math/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_chat_math.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_chat_math:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-chat-math/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-chat-math/README.md -o examples/flows/evaluation/eval-chat-math
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-chat-math
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-chat-math/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_classification_accuracy.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_classification_accuracy
+on:
+  schedule:
+    - cron: "39 22 * * *" # Every day starting at 6:39 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-classification-accuracy/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_classification_accuracy.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_classification_accuracy:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-classification-accuracy/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-classification-accuracy/README.md -o examples/flows/evaluation/eval-classification-accuracy
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-classification-accuracy
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-classification-accuracy/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_entity_match_rate.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_entity_match_rate
+on:
+  schedule:
+    - cron: "10 22 * * *" # Every day starting at 6:10 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-entity-match-rate/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_entity_match_rate.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_entity_match_rate:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-entity-match-rate/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-entity-match-rate/README.md -o examples/flows/evaluation/eval-entity-match-rate
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-entity-match-rate
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-entity-match-rate/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_groundedness.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_groundedness
+on:
+  schedule:
+    - cron: "48 22 * * *" # Every day starting at 6:48 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-groundedness/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_groundedness.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_groundedness:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-groundedness/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-groundedness/README.md -o examples/flows/evaluation/eval-groundedness
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-groundedness
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-groundedness/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_multi_turn_metrics
+on:
+  schedule:
+    - cron: "7 21 * * *" # Every day starting at 5:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-multi-turn-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_multi_turn_metrics:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-multi-turn-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-multi-turn-metrics/README.md -o examples/flows/evaluation/eval-multi-turn-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-multi-turn-metrics/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_perceived_intelligence.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_perceived_intelligence
+on:
+  schedule:
+    - cron: "41 21 * * *" # Every day starting at 5:41 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-perceived-intelligence/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_perceived_intelligence.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_perceived_intelligence:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-perceived-intelligence/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-perceived-intelligence/README.md -o examples/flows/evaluation/eval-perceived-intelligence
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-perceived-intelligence
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-perceived-intelligence/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_qna_non_rag.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_qna_non_rag
+on:
+  schedule:
+    - cron: "6 20 * * *" # Every day starting at 4:6 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-qna-non-rag/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_qna_non_rag.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_qna_non_rag:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-qna-non-rag/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-qna-non-rag/README.md -o examples/flows/evaluation/eval-qna-non-rag
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-qna-non-rag
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-qna-non-rag/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_qna_rag_metrics.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_qna_rag_metrics
+on:
+  schedule:
+    - cron: "55 19 * * *" # Every day starting at 3:55 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-qna-rag-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_qna_rag_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_qna_rag_metrics:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-qna-rag-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-qna-rag-metrics/README.md -o examples/flows/evaluation/eval-qna-rag-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-qna-rag-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-qna-rag-metrics/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_single_turn_metrics.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_single_turn_metrics
+on:
+  schedule:
+    - cron: "32 19 * * *" # Every day starting at 3:32 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-single-turn-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_single_turn_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_single_turn_metrics:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-single-turn-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-single-turn-metrics/README.md -o examples/flows/evaluation/eval-single-turn-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-single-turn-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-single-turn-metrics/bash_script.sh

.github/workflows/samples_flows_evaluation_eval_summarization.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_summarization
+on:
+  schedule:
+    - cron: "57 19 * * *" # Every day starting at 3:57 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-summarization/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_summarization.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_evaluation_eval_summarization:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/evaluation/eval-summarization/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-summarization/README.md -o examples/flows/evaluation/eval-summarization
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-summarization
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-summarization/bash_script.sh

.github/workflows/samples_flows_standard_autonomous_agent.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_autonomous_agent
+on:
+  schedule:
+    - cron: "11 20 * * *" # Every day starting at 4:11 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/autonomous-agent/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_autonomous_agent.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_autonomous_agent:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/autonomous-agent/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/autonomous-agent/README.md -o examples/flows/standard/autonomous-agent
+      - name: Cat script
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/autonomous-agent
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/autonomous-agent/bash_script.sh

.github/workflows/samples_flows_standard_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_basic
+on:
+  schedule:
+    - cron: "41 21 * * *" # Every day starting at 5:41 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/basic/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/basic/README.md -o examples/flows/standard/basic
+      - name: Cat script
+        working-directory: examples/flows/standard/basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/basic/bash_script.sh

.github/workflows/samples_flows_standard_basic_with_builtin_llm.yml

@@ -0,0 +1,126 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_basic_with_builtin_llm
+on:
+  schedule:
+    - cron: "14 21 * * *" # Every day starting at 5:14 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/basic-with-builtin-llm/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_basic_with_builtin_llm.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_basic_with_builtin_llm:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Create AOAI Connection from YAML
+        working-directory: ${{ github.workspace }}
+        run: pf connection create --file examples/connections/azure_openai.yml --set api_key=${{ secrets.AOAI_API_KEY_TEST }} api_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/basic-with-builtin-llm/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/basic-with-builtin-llm/README.md -o examples/flows/standard/basic-with-builtin-llm
+      - name: Cat script
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/basic-with-builtin-llm
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/basic-with-builtin-llm/bash_script.sh

.github/workflows/samples_flows_standard_basic_with_connection.yml

@@ -0,0 +1,126 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_basic_with_connection
+on:
+  schedule:
+    - cron: "11 20 * * *" # Every day starting at 4:11 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/basic-with-connection/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_basic_with_connection.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_basic_with_connection:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Create AOAI Connection from YAML
+        working-directory: ${{ github.workspace }}
+        run: pf connection create --file examples/connections/azure_openai.yml --set api_key=${{ secrets.AOAI_API_KEY_TEST }} api_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/basic-with-connection/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/basic-with-connection/README.md -o examples/flows/standard/basic-with-connection
+      - name: Cat script
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/basic-with-connection
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/basic-with-connection/bash_script.sh

.github/workflows/samples_flows_standard_conditional_flow_for_if_else.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_conditional_flow_for_if_else
+on:
+  schedule:
+    - cron: "8 21 * * *" # Every day starting at 5:8 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/conditional-flow-for-if-else/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_conditional_flow_for_if_else.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_conditional_flow_for_if_else:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/conditional-flow-for-if-else/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/conditional-flow-for-if-else/README.md -o examples/flows/standard/conditional-flow-for-if-else
+      - name: Cat script
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/conditional-flow-for-if-else
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/conditional-flow-for-if-else/bash_script.sh

.github/workflows/samples_flows_standard_conditional_flow_for_switch.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_conditional_flow_for_switch
+on:
+  schedule:
+    - cron: "21 20 * * *" # Every day starting at 4:21 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/conditional-flow-for-switch/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_conditional_flow_for_switch.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_conditional_flow_for_switch:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/conditional-flow-for-switch/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/conditional-flow-for-switch/README.md -o examples/flows/standard/conditional-flow-for-switch
+      - name: Cat script
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/conditional-flow-for-switch
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/conditional-flow-for-switch/bash_script.sh

.github/workflows/samples_flows_standard_customer_intent_extraction.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_customer_intent_extraction
+on:
+  schedule:
+    - cron: "17 21 * * *" # Every day starting at 5:17 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/customer-intent-extraction/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_customer_intent_extraction.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_customer_intent_extraction:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/customer-intent-extraction/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/customer-intent-extraction/README.md -o examples/flows/standard/customer-intent-extraction
+      - name: Cat script
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/customer-intent-extraction
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/customer-intent-extraction/bash_script.sh

.github/workflows/samples_flows_standard_describe_image.yml

@@ -0,0 +1,121 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_describe_image
+on:
+  schedule:
+    - cron: "9 19 * * *" # Every day starting at 3:9 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/describe-image/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_describe_image.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_describe_image:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_GPT_4V_KEY }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          cp ../../../connections/azure_openai.yml ./azure_openai.yml
+          sed -i -e "s/<user-input>/$AOAI_API_KEY/g" -e "s/aoai-api-endpoint/$AOAI_API_ENDPOINT/g" azure_openai.yml
+      - name: Create AOAI Connection from ENV file
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          if [[ -e .env ]]; then
+            pf connection create --file .env --name aoai_gpt4v_connection
+          fi
+          if [[ -e azure_openai.yml ]]; then
+            pf connection create --file azure_openai.yml --name aoai_gpt4v_connection
+          fi
+          pf connection list
+      - name: Create run.yml
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/describe-image/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/describe-image/README.md -o examples/flows/standard/describe-image
+      - name: Cat script
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          export aoai_api_key=${{secrets.AOAI_GPT_4V_KEY }}
+          export aoai_api_endpoint=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          export aoai_api_key=${{secrets.AOAI_GPT_4V_KEY }}
+          export aoai_api_endpoint=${{ secrets.AOAI_GPT_4V_ENDPOINT }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/describe-image
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/describe-image/bash_script.sh

.github/workflows/samples_flows_standard_flow_with_additional_includes.yml

@@ -0,0 +1,126 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_flow_with_additional_includes
+on:
+  schedule:
+    - cron: "29 20 * * *" # Every day starting at 4:29 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/flow-with-additional-includes/**, examples/*requirements.txt, examples/flows/standard/web-classification/**, .github/workflows/samples_flows_standard_flow_with_additional_includes.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_flow_with_additional_includes:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Create AOAI Connection from YAML
+        working-directory: ${{ github.workspace }}
+        run: pf connection create --file examples/connections/azure_openai.yml --set api_key=${{ secrets.AOAI_API_KEY_TEST }} api_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/flow-with-additional-includes/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/flow-with-additional-includes/README.md -o examples/flows/standard/flow-with-additional-includes
+      - name: Cat script
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/flow-with-additional-includes
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/flow-with-additional-includes/bash_script.sh

.github/workflows/samples_flows_standard_flow_with_symlinks.yml

@@ -0,0 +1,126 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_flow_with_symlinks
+on:
+  schedule:
+    - cron: "41 20 * * *" # Every day starting at 4:41 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/flow-with-symlinks/**, examples/*requirements.txt, examples/flows/standard/web-classification/**, .github/workflows/samples_flows_standard_flow_with_symlinks.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_flow_with_symlinks:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Create AOAI Connection from YAML
+        working-directory: ${{ github.workspace }}
+        run: pf connection create --file examples/connections/azure_openai.yml --set api_key=${{ secrets.AOAI_API_KEY_TEST }} api_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/flow-with-symlinks/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/flow-with-symlinks/README.md -o examples/flows/standard/flow-with-symlinks
+      - name: Cat script
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/flow-with-symlinks
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/flow-with-symlinks/bash_script.sh

.github/workflows/samples_flows_standard_gen_docstring.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_gen_docstring
+on:
+  schedule:
+    - cron: "0 20 * * *" # Every day starting at 4:0 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/gen-docstring/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_gen_docstring.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_gen_docstring:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/gen-docstring/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/gen-docstring/README.md -o examples/flows/standard/gen-docstring
+      - name: Cat script
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/gen-docstring
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/gen-docstring/bash_script.sh

.github/workflows/samples_flows_standard_maths_to_code.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_maths_to_code
+on:
+  schedule:
+    - cron: "39 19 * * *" # Every day starting at 3:39 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/maths-to-code/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_maths_to_code.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_maths_to_code:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/maths-to-code/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/maths-to-code/README.md -o examples/flows/standard/maths-to-code
+      - name: Cat script
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/maths-to-code
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/maths-to-code/bash_script.sh

.github/workflows/samples_flows_standard_named_entity_recognition.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_named_entity_recognition
+on:
+  schedule:
+    - cron: "11 22 * * *" # Every day starting at 6:11 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/named-entity-recognition/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_named_entity_recognition.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_named_entity_recognition:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/named-entity-recognition/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/named-entity-recognition/README.md -o examples/flows/standard/named-entity-recognition
+      - name: Cat script
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/named-entity-recognition
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/named-entity-recognition/bash_script.sh

.github/workflows/samples_flows_standard_question_simulation.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_question_simulation
+on:
+  schedule:
+    - cron: "37 19 * * *" # Every day starting at 3:37 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/question-simulation/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_question_simulation.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_question_simulation:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/question-simulation/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/question-simulation/README.md -o examples/flows/standard/question-simulation
+      - name: Cat script
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/question-simulation
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/question-simulation/bash_script.sh

.github/workflows/samples_flows_standard_web_classification.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_standard_web_classification
+on:
+  schedule:
+    - cron: "13 21 * * *" # Every day starting at 5:13 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/standard/web-classification/**, examples/*requirements.txt, .github/workflows/samples_flows_standard_web_classification.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_flows_standard_web_classification:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/flows/standard/web-classification/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/standard/web-classification/README.md -o examples/flows/standard/web-classification
+      - name: Cat script
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/standard/web-classification
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/flows/standard/web-classification/bash_script.sh

.github/workflows/samples_getstarted_flowasfunction.yml

@@ -0,0 +1,74 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_getstarted_flowasfunction
+on:
+  schedule:
+    - cron: "17 21 * * *" # Every day starting at 5:17 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/get-started/**, examples/*requirements.txt, .github/workflows/samples_getstarted_flowasfunction.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_getstarted_flowasfunction:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Create new Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}" name=new_ai_connection
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/get-started
+        run: |
+          papermill -k python flow-as-function.ipynb flow-as-function.output.ipynb -p api_key ${{ secrets.AOAI_API_KEY_TEST }} -p api_base ${{ secrets.AOAI_API_ENDPOINT_TEST }} -p api_version 2023-07-01-preview
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/get-started

.github/workflows/samples_getstarted_quickstart.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_getstarted_quickstart
+on:
+  schedule:
+    - cron: "55 21 * * *" # Every day starting at 5:55 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/get-started/**, examples/*requirements.txt, .github/workflows/samples_getstarted_quickstart.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_getstarted_quickstart:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/tutorials/get-started
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/get-started
+        run: |
+          papermill -k python quickstart.ipynb quickstart.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/get-started

.github/workflows/samples_getstarted_quickstartazure.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_getstarted_quickstartazure
+on:
+  schedule:
+    - cron: "24 20 * * *" # Every day starting at 4:24 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/get-started/**, examples/*requirements.txt, .github/workflows/samples_getstarted_quickstartazure.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_getstarted_quickstartazure:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/get-started
+        run: |
+          papermill -k python quickstart-azure.ipynb quickstart-azure.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/get-started

.github/workflows/samples_prompty_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_basic
+on:
+  schedule:
+    - cron: "46 19 * * *" # Every day starting at 3:46 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/basic/**, examples/*requirements.txt, .github/workflows/samples_prompty_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/prompty/basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/prompty/basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/prompty/basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/prompty/basic/README.md -o examples/prompty/basic
+      - name: Cat script
+        working-directory: examples/prompty/basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/prompty/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/prompty/basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/prompty/basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/prompty/basic/bash_script.sh

.github/workflows/samples_prompty_basic_promptyquickstart.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_basic_promptyquickstart
+on:
+  schedule:
+    - cron: "19 21 * * *" # Every day starting at 5:19 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/basic/**, examples/*requirements.txt, .github/workflows/samples_prompty_basic_promptyquickstart.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_basic_promptyquickstart:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/prompty/basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/prompty/basic
+        run: |
+          papermill -k python prompty-quickstart.ipynb prompty-quickstart.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/prompty/basic

.github/workflows/samples_prompty_chat_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_chat_basic
+on:
+  schedule:
+    - cron: "47 20 * * *" # Every day starting at 4:47 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_prompty_chat_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_chat_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/prompty/chat-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/prompty/chat-basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/prompty/chat-basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/prompty/chat-basic/README.md -o examples/prompty/chat-basic
+      - name: Cat script
+        working-directory: examples/prompty/chat-basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/prompty/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/prompty/chat-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/prompty/chat-basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/prompty/chat-basic/bash_script.sh

.github/workflows/samples_prompty_chatbasic_chatwithprompty.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_chatbasic_chatwithprompty
+on:
+  schedule:
+    - cron: "51 19 * * *" # Every day starting at 3:51 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/chat-basic/**, examples/*requirements.txt, .github/workflows/samples_prompty_chatbasic_chatwithprompty.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_chatbasic_chatwithprompty:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/prompty/chat-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/prompty/chat-basic
+        run: |
+          papermill -k python chat-with-prompty.ipynb chat-with-prompty.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/prompty/chat-basic

.github/workflows/samples_prompty_eval_apology.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_eval_apology
+on:
+  schedule:
+    - cron: "10 22 * * *" # Every day starting at 6:10 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/eval-apology/**, examples/*requirements.txt, .github/workflows/samples_prompty_eval_apology.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_eval_apology:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/prompty/eval-apology
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/prompty/eval-apology
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/prompty/eval-apology/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/prompty/eval-apology/README.md -o examples/prompty/eval-apology
+      - name: Cat script
+        working-directory: examples/prompty/eval-apology
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/prompty/eval-apology
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/prompty/eval-apology
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/prompty/eval-apology
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/prompty/eval-apology/bash_script.sh

.github/workflows/samples_prompty_eval_basic.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_eval_basic
+on:
+  schedule:
+    - cron: "25 21 * * *" # Every day starting at 5:25 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/eval-basic/**, examples/*requirements.txt, .github/workflows/samples_prompty_eval_basic.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_eval_basic:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/prompty/eval-basic
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/prompty/eval-basic
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/prompty/eval-basic/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/prompty/eval-basic/README.md -o examples/prompty/eval-basic
+      - name: Cat script
+        working-directory: examples/prompty/eval-basic
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/prompty/eval-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/prompty/eval-basic
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/prompty/eval-basic
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/prompty/eval-basic/bash_script.sh

.github/workflows/samples_prompty_format_output.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_format_output
+on:
+  schedule:
+    - cron: "9 22 * * *" # Every day starting at 6:9 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/format-output/**, examples/*requirements.txt, .github/workflows/samples_prompty_format_output.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_format_output:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/prompty/format-output
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/prompty/format-output
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/prompty/format-output/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/prompty/format-output/README.md -o examples/prompty/format-output
+      - name: Cat script
+        working-directory: examples/prompty/format-output
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/prompty/format-output
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/prompty/format-output
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/prompty/format-output
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/prompty/format-output/bash_script.sh

.github/workflows/samples_prompty_formatoutput_promptyoutputformat.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_prompty_formatoutput_promptyoutputformat
+on:
+  schedule:
+    - cron: "7 20 * * *" # Every day starting at 4:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/prompty/format-output/**, examples/*requirements.txt, .github/workflows/samples_prompty_formatoutput_promptyoutputformat.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_prompty_formatoutput_promptyoutputformat:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/prompty/format-output
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/prompty/format-output
+        run: |
+          papermill -k python prompty-output-format.ipynb prompty-output-format.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/prompty/format-output

.github/workflows/samples_runflowwithpipeline_pipeline.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_runflowwithpipeline_pipeline
+on:
+  schedule:
+    - cron: "33 20 * * *" # Every day starting at 4:33 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/run-flow-with-pipeline/**, examples/*requirements.txt, .github/workflows/samples_runflowwithpipeline_pipeline.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_runflowwithpipeline_pipeline:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/run-flow-with-pipeline
+        run: |
+          papermill -k python pipeline.ipynb pipeline.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/run-flow-with-pipeline

.github/workflows/samples_runmanagement_cloudrunmanagement.yml

@@ -0,0 +1,78 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_runmanagement_cloudrunmanagement
+on:
+  schedule:
+    - cron: "24 20 * * *" # Every day starting at 4:24 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/run-management/**, examples/*requirements.txt, .github/workflows/samples_runmanagement_cloudrunmanagement.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_runmanagement_cloudrunmanagement:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Generate config.json for canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Generate config.json for production workspace
+        if: github.event_name != 'schedule'
+        run: echo '${{ secrets.EXAMPLE_WORKSPACE_CONFIG_JSON_PROD }}' > ${{ github.workspace }}/examples/config.json
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/run-management
+        run: |
+          papermill -k python cloud-run-management.ipynb cloud-run-management.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/run-management

.github/workflows/samples_runmanagement_runmanagement.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_runmanagement_runmanagement
+on:
+  schedule:
+    - cron: "51 20 * * *" # Every day starting at 4:51 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/run-management/**, examples/*requirements.txt, .github/workflows/samples_runmanagement_runmanagement.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_runmanagement_runmanagement:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/tutorials/run-management
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/run-management
+        run: |
+          papermill -k python run-management.ipynb run-management.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/run-management

.github/workflows/samples_tools_use_cases_cascading_inputs_tool_showcase.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tools_use_cases_cascading_inputs_tool_showcase
+on:
+  schedule:
+    - cron: "41 20 * * *" # Every day starting at 4:41 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tools/use-cases/cascading-inputs-tool-showcase/**, examples/*requirements.txt, .github/workflows/samples_tools_use_cases_cascading_inputs_tool_showcase.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tools_use_cases_cascading_inputs_tool_showcase:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/tools/use-cases/cascading-inputs-tool-showcase/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/tools/use-cases/cascading-inputs-tool-showcase/README.md -o examples/tools/use-cases/cascading-inputs-tool-showcase
+      - name: Cat script
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/tools/use-cases/cascading-inputs-tool-showcase
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/tools/use-cases/cascading-inputs-tool-showcase/bash_script.sh

.github/workflows/samples_tools_use_cases_custom_llm_tool_showcase.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tools_use_cases_custom_llm_tool_showcase
+on:
+  schedule:
+    - cron: "40 21 * * *" # Every day starting at 5:40 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tools/use-cases/custom_llm_tool_showcase/**, examples/*requirements.txt, .github/workflows/samples_tools_use_cases_custom_llm_tool_showcase.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tools_use_cases_custom_llm_tool_showcase:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/tools/use-cases/custom_llm_tool_showcase/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/tools/use-cases/custom_llm_tool_showcase/README.md -o examples/tools/use-cases/custom_llm_tool_showcase
+      - name: Cat script
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/tools/use-cases/custom_llm_tool_showcase
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/tools/use-cases/custom_llm_tool_showcase/bash_script.sh

.github/workflows/samples_tools_use_cases_custom_strong_type_connection_package_tool_showcase.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tools_use_cases_custom_strong_type_connection_package_tool_showcase
+on:
+  schedule:
+    - cron: "0 20 * * *" # Every day starting at 4:0 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase/**, examples/*requirements.txt, .github/workflows/samples_tools_use_cases_custom_strong_type_connection_package_tool_showcase.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tools_use_cases_custom_strong_type_connection_package_tool_showcase:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase/README.md -o examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+      - name: Cat script
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/tools/use-cases/custom-strong-type-connection-package-tool-showcase/bash_script.sh

.github/workflows/samples_tools_use_cases_custom_strong_type_connection_script_tool_showcase.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tools_use_cases_custom_strong_type_connection_script_tool_showcase
+on:
+  schedule:
+    - cron: "58 20 * * *" # Every day starting at 4:58 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase/**, examples/*requirements.txt, .github/workflows/samples_tools_use_cases_custom_strong_type_connection_script_tool_showcase.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tools_use_cases_custom_strong_type_connection_script_tool_showcase:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase/README.md -o examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+      - name: Cat script
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/tools/use-cases/custom-strong-type-connection-script-tool-showcase/bash_script.sh

.github/workflows/samples_tools_use_cases_dynamic_list_input_tool_showcase.yml

@@ -0,0 +1,123 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tools_use_cases_dynamic_list_input_tool_showcase
+on:
+  schedule:
+    - cron: "18 20 * * *" # Every day starting at 4:18 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tools/use-cases/dynamic-list-input-tool-showcase/**, examples/*requirements.txt, .github/workflows/samples_tools_use_cases_dynamic_list_input_tool_showcase.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tools_use_cases_dynamic_list_input_tool_showcase:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 0
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Extract Steps examples/tools/use-cases/dynamic-list-input-tool-showcase/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/tools/use-cases/dynamic-list-input-tool-showcase/README.md -o examples/tools/use-cases/dynamic-list-input-tool-showcase
+      - name: Cat script
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/tools/use-cases/dynamic-list-input-tool-showcase
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact
+          path: examples/tools/use-cases/dynamic-list-input-tool-showcase/bash_script.sh

.github/workflows/samples_tracing_autogengroupchat_traceautogengroupchat.yml

@@ -0,0 +1,93 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tracing_autogengroupchat_traceautogengroupchat
+on:
+  schedule:
+    - cron: "11 20 * * *" # Every day starting at 4:11 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/tracing/autogen-groupchat/**, examples/*requirements.txt, .github/workflows/samples_tracing_autogengroupchat_traceautogengroupchat.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tracing_autogengroupchat_traceautogengroupchat:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/tutorials/tracing/autogen-groupchat
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+          if [[ -e OAI_CONFIG_LIST.json.example ]]; then
+            echo "OAI_CONFIG_LIST replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" OAI_CONFIG_LIST.json.example
+            mv OAI_CONFIG_LIST.json.example OAI_CONFIG_LIST.json
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/tracing/autogen-groupchat
+        run: |
+          papermill -k python trace-autogen-groupchat.ipynb trace-autogen-groupchat.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/tracing/autogen-groupchat

.github/workflows/samples_tracing_customotlpcollector_otlptracecollector.yml

@@ -0,0 +1,88 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_tracing_customotlpcollector_otlptracecollector
+on:
+  schedule:
+    - cron: "22 21 * * *" # Every day starting at 5:22 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/tutorials/tracing/custom-otlp-collector/**, examples/*requirements.txt, .github/workflows/samples_tracing_customotlpcollector_otlptracecollector.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  samples_tracing_customotlpcollector_otlptracecollector:
+    runs-on: ubuntu-latest
+    environment:
+      internal
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/tutorials/tracing/custom-otlp-collector
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Random Wait
+        uses: AliSajid/random-wait-action@main
+        with:
+          minimum: 1
+          maximum: 99
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}}
+          tenant-id: ${{secrets.AZURE_TENANT_ID}}
+          client-id: ${{secrets.AZURE_CLIENT_ID}}
+      - name: Fetch OID token every 4 mins
+        shell: bash
+        run: |
+          while true; do
+            token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN
+            token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL
+            token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r)
+            az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -t ${{secrets.AZURE_TENANT_ID}} --federated-token $token --output none
+            # Sleep for 4 minutes
+            sleep 240
+          done &
+      - name: Test Notebook
+        working-directory: examples/tutorials/tracing/custom-otlp-collector
+        run: |
+          papermill -k python otlp-trace-collector.ipynb otlp-trace-collector.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/tracing/custom-otlp-collector