chore: import upstream snapshot with attribution

examples/flows/integrations/unify-ai/.env.example

@@ -0,0 +1,2 @@
+UNIFY_AI_API_KEY=<your_Unify_AI_api_key>
+UNIFY_AI_BASE_URL=https://api.unify.ai/v0/ #Please refer https://unify.ai/docs/concepts/unify_api.html

examples/flows/integrations/unify-ai/README.md

@@ -0,0 +1,107 @@
+# Basic standard flow with Unify AI
+A basic standard flow define using function entry that calls Unify AI. 
+
+Unify AI helps you use a LLM from a wide variety of models and providers using a single Unify API key. You can make an optimal choice by comparing trade-offs between quality, cost and latency. 
+
+Refer [Unify AI documentation](https://unify.ai/docs).
+
+## Prerequisites
+
+Install promptflow sdk and other dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+## Run flow
+
+- Prepare your Unify AI account follow this [instruction](https://unify.ai/docs/index.html#getting-started) and get your `api_key` if you don't have one.
+
+- Setup environment variables
+
+Ensure you have put your Unify key in [.env](./.env) file. You can create one refer to this [example file](./.env.example).
+
+```bash
+cat ./.env
+```
+
+- Run/Debug as normal Python file
+```bash
+python programmer.py
+```
+
+- Test with flow entry
+```bash
+pf flow test --flow programmer:write_simple_program --inputs text="Java Hello World!"
+```
+
+- Test with flow yaml
+```bash
+# test with sample input value in flow.flex.yaml
+pf flow test --flow .
+```
+
+```shell
+# test with UI
+pf flow test --flow . --ui
+```
+
+- Create run with multiple lines data
+```bash
+# using environment from .env file (loaded in user code: hello.py)
+pf run create --flow . --data ./data.jsonl --column-mapping text='${data.text}' --stream
+```
+
+You can also skip providing `column-mapping` if provided data has same column name as the flow.
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.
+
+- List and show run meta
+```bash
+# list created run
+pf run list
+
+# get a sample run name
+
+name=$(pf run list -r 10 | jq '.[] | select(.name | contains("basic_")) | .name'| head -n 1 | tr -d '"')
+# show specific run detail
+pf run show --name $name
+
+# show output
+pf run show-details --name $name
+
+# visualize run in browser
+pf run visualize --name $name
+```
+
+## Run flow in cloud with connection
+
+```bash
+# set default workspace
+az account set -s <your_subscription_id>
+az configure --defaults group=<your_resource_group_name> workspace=<your_workspace_name>
+```
+
+- Create run
+```bash
+# run with environment variable reference connection in azureml workspace
+pfazure run create --flow . --data ./data.jsonl --column-mapping text='${data.text}' --environment-variables UNIFY_AI_API_KEY='<unify_api_key>' UNIFY_AI_BASE_URL='https://api.unify.ai/v0/' --stream
+# run using yaml file
+pfazure run create --file run.yml --stream
+```
+
+- List and show run meta
+```bash
+# list created run
+pfazure run list -r 3
+
+# get a sample run name
+name=$(pfazure run list -r 100 | jq '.[] | select(.name | contains("basic_")) | .name'| head -n 1 | tr -d '"')
+
+# show specific run detail
+pfazure run show --name $name
+
+# show output
+pfazure run show-details --name $name
+
+# visualize run in browser
+pfazure run visualize --name $name
+```

examples/flows/integrations/unify-ai/data.jsonl

@@ -0,0 +1,3 @@
+{"text": "Python Hello World!"}
+{"text": "C Hello World!"}
+{"text": "C# Hello World!"}

examples/flows/integrations/unify-ai/eval_code_flow/code-eval-flow.flex.yaml

@@ -0,0 +1,6 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+# flow is defined as python function
+entry: code_quality_unify_ai:CodeEvaluator
+environment:
+  # image: mcr.microsoft.com/azureml/promptflow/promptflow-python
+  python_requirements_txt: requirements.txt

examples/flows/integrations/unify-ai/eval_code_flow/code_quality_unify_ai.py

@@ -0,0 +1,61 @@
+import json
+from pathlib import Path
+from typing import TypedDict
+
+from jinja2 import Template
+
+from promptflow.core import OpenAIModelConfiguration
+from promptflow.core._flow import Prompty
+from promptflow.tracing import trace
+
+BASE_DIR = Path(__file__).absolute().parent
+
+# Derived from https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/eval-code-quality/
+
+
+@trace
+def load_prompt(jinja2_template: str, code: str, examples: list) -> str:
+    """Load prompt function."""
+    with open(BASE_DIR / jinja2_template, "r", encoding="utf-8") as f:
+        tmpl = Template(f.read(), trim_blocks=True, keep_trailing_newline=True)
+        prompt = tmpl.render(code=code, examples=examples)
+        return prompt
+
+
+class Result(TypedDict):
+    correctness: float
+    readability: float
+    explanation: str
+
+
+class CodeEvaluator:
+    """ Uses Unify AI's LLM to evaluate a code block.
+    Note:
+    OpenAI client is being repurposed to call Unify AI API, Since Unify AI API is competable with OpenAI API.
+    This enables reusing Promptflow's OpenAI integration/support with Unify AI.
+
+    """
+    def __init__(self, model_config: OpenAIModelConfiguration):
+        self.model_config = model_config
+
+    def __call__(self, code: str) -> Result:
+        """Evaluate the code based on correctness, readability."""
+        prompty = Prompty.load(
+            source=BASE_DIR / "eval_code_quality.prompty",
+            model={"configuration": self.model_config},
+        )
+        output = prompty(code=code)
+        output = json.loads(output)
+        output = Result(**output)
+        return output
+
+    def __aggregate__(self, line_results: list) -> dict:
+        """Aggregate the results."""
+        total = len(line_results)
+        avg_correctness = sum(int(r["correctness"]) for r in line_results) / total
+        avg_readability = sum(int(r["readability"]) for r in line_results) / total
+        return {
+            "average_correctness": avg_correctness,
+            "average_readability": avg_readability,
+            "total": total,
+        }

examples/flows/integrations/unify-ai/eval_code_flow/eval_code_quality.prompty

@@ -0,0 +1,39 @@
+---
+name: Evaluate code quality
+description: Evaluate the quality of code snippet.
+model:
+  api: chat
+  configuration:
+    type: unify
+    model_name: llama-3.1-8b-chat
+    provider_name: together-ai
+  parameters:
+    temperature: 0.2
+inputs: 
+  code:
+    type: string
+sample: ${file:sample.json}
+---
+# system:
+You are an AI assistant. 
+You task is to evaluate the code based on correctness, readability.
+Only accepts valid JSON format response without extra prefix or postfix.
+
+# user:
+This correctness value should always be an integer between 1 and 5. So the correctness produced should be 1 or 2 or 3 or 4 or 5.
+This readability value should always be an integer between 1 and 5. So the readability produced should be 1 or 2 or 3 or 4 or 5.
+
+Here are a few examples:
+
+**Example 1**
+Code: print(\"Hello, world!\")
+OUTPUT:
+{
+    "correctness": 5,
+    "readability": 5,
+    "explanation": "The code is correct as it is a simple question and answer format. The readability is also good as the code is short and easy to understand."
+}
+
+For a given code, valuate the code based on correctness, readability:
+Code: {{code}}
+OUTPUT:

examples/flows/integrations/unify-ai/eval_code_flow/sample.json

@@ -0,0 +1,3 @@
+{
+    "code": "print(\"Hello, world!\")"
+}

examples/flows/integrations/unify-ai/flex-flow-quickstart-unify-ai.ipynb

@@ -0,0 +1,408 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Getting started with flex flow using Unify AI"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "**Learning Objectives** - Upon completing this tutorial, you should be able to:\n",
+    "\n",
+    "- Write LLM application using Unify AI API in notebook and visualize the trace of your application.\n",
+    "- Choose a model/provider from [Unify model catalogue](https://unify.ai/benchmarks) for your flex flow.\n",
+    "- Convert the application into a flow and batch run against multi lines of data.\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 0. Install dependent packages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture --no-stderr\n",
+    "%pip install -r ./requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Trace your application with promptflow\n",
+    "\n",
+    "Assume we already have a python function that calls Unify AI. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"llm.py\") as fin:\n",
+    "    print(fin.read())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note: before running below cell, please configure required environment variable `UNIFY_AI_API_KEY` by create an `.env` file. Please refer to `./.env.example` as an template."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Choose LLM model and provider\n",
+    "\n",
+    "Define provider and model from Unify AI. Refer to [Unify model catalogue](https://unify.ai/benchmarks)\n",
+    "\n",
+    "Choose an optimal model/provider combination for your usecase by comparing trade-offs between quality, cost and latency. \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model_name and provider_name defined here are used throughout the notebook.\n",
+    "# This example use llama 3.1 8b params and together-ai, redefine as per your usecase.\n",
+    "\n",
+    "model_name = \"llama-3.1-8b-chat\"\n",
+    "provider_name = \"together-ai\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "from llm import my_llm_tool\n",
+    "\n",
+    "# pls configure `UNIFY_AI_API_KEY` environment variable\n",
+    "result = my_llm_tool(\n",
+    "    prompt=\"Write a simple Hello, world! python program that displays the greeting message when executed. Output code only.\",\n",
+    "    model_name=model_name,\n",
+    "    provider_name=provider_name,\n",
+    ")\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Visualize trace by using start_trace\n",
+    "\n",
+    "Note we add `@trace` in the `my_llm_tool` function, re-run below cell will collect a trace in trace UI."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from promptflow.tracing import start_trace\n",
+    "\n",
+    "# start a trace session, and print a url for user to check trace\n",
+    "start_trace()\n",
+    "# rerun the function, which will be recorded in the trace\n",
+    "result = my_llm_tool(\n",
+    "    prompt=\"Write a simple Hello, world! program that displays the greeting message when executed. Output code only.\",\n",
+    "    model_name=model_name,\n",
+    "    provider_name=provider_name,\n",
+    ")\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, let's add another layer of function call. In `programmer.py` there is a function called `write_simple_program`, which calls a new function called `load_prompt` and previous `my_llm_tool` function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# show the programmer.py content\n",
+    "with open(\"programmer.py\") as fin:\n",
+    "    print(fin.read())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# call the flow entry function\n",
+    "from programmer import write_simple_program\n",
+    "\n",
+    "result = write_simple_program(\"Java Hello, world!\")\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Setup model configuration with environment variables\n",
+    "\n",
+    "When used in local, create a model configuration object with environment variables."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note: before running below cell, please configure required environment variable `UNIFY_AI_API_KEY` and `UNIFY_AI_BASE_URL` by creating a `.env` file. Please refer to `./.env.example` as an template.\n",
+    "\n",
+    "Here OpenAI client is being used to call Unify AI API. `UNIFY_AI_BASE_URL` is the base url for the API endpoint (along with version) in [Unify API Documentation](https://unify.ai/docs/concepts/unify_api.html). `./.env.example` contains base url for for Unify API version v0."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "from promptflow.core import OpenAIModelConfiguration\n",
+    "\n",
+    "# pls configure `UNIFY_AI_API_KEY`, `UNIFY_AI_BASE_URL` environment variables first\n",
+    "if \"UNIFY_AI_API_KEY\" not in os.environ:\n",
+    "    # load environment variables from .env file\n",
+    "    load_dotenv()\n",
+    "\n",
+    "if \"UNIFY_AI_API_KEY\" not in os.environ:\n",
+    "    raise Exception(\"Please specify environment variables: UNIFY_AI_API_KEY\")\n",
+    "model_config = OpenAIModelConfiguration(\n",
+    "    base_url=os.environ[\"UNIFY_AI_BASE_URL\"],\n",
+    "    api_key=os.environ[\"UNIFY_AI_API_KEY\"],\n",
+    "    model=f\"{model_name}@{provider_name}\" \n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Eval the result "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "from eval_code_flow.code_quality_unify_ai import CodeEvaluator\n",
+    "\n",
+    "\n",
+    "evaluator = CodeEvaluator(model_config=model_config)\n",
+    "eval_result = evaluator(result)\n",
+    "eval_result\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Batch run the function as flow with multi-line data\n",
+    "\n",
+    "Create a [flow.flex.yaml](https://github.com/microsoft/promptflow/blob/main/examples/flex-flows/basic/flow.flex.yaml) file to define a flow which entry pointing to the python function we defined.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# show the flow.flex.yaml content\n",
+    "with open(\"flow.flex.yaml\") as fin:\n",
+    "    print(fin.read())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Batch run with a data file (with multiple lines of test data)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from promptflow.client import PFClient\n",
+    "\n",
+    "pf = PFClient()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = \"./data.jsonl\"  # path to the data file\n",
+    "# create run with the flow function and data\n",
+    "base_run = pf.run(\n",
+    "    flow=write_simple_program,\n",
+    "    data=data,\n",
+    "    column_mapping={\n",
+    "        \"text\": \"${data.text}\",\n",
+    "    },\n",
+    "    stream=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "details = pf.get_details(base_run)\n",
+    "details.head(10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Evaluate your flow\n",
+    "Then you can use an evaluation method to evaluate your flow. The evaluation methods are also flows which usually using LLM assert the produced output matches certain expectation. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Run evaluation on the previous batch run\n",
+    "The **base_run** is the batch run we completed in step 2 above, for web-classification flow with \"data.jsonl\" as input."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# we can also run flow pointing to code evaluator yaml file\n",
+    "eval_flow = \"./eval_code_flow/code-eval-flow.flex.yaml\"\n",
+    "\n",
+    "eval_run = pf.run(\n",
+    "    flow=eval_flow,\n",
+    "    init={\"model_config\": model_config},\n",
+    "    data=\"./data.jsonl\",  # path to the data file\n",
+    "    run=base_run,  # specify base_run as the run you want to evaluate\n",
+    "    column_mapping={\n",
+    "        \"code\": \"${run.outputs.output}\",\n",
+    "    },\n",
+    "    stream=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "details = pf.get_details(eval_run)\n",
+    "details.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "metrics = pf.get_metrics(eval_run)\n",
+    "print(json.dumps(metrics, indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pf.visualize([base_run, eval_run])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## End Note\n",
+    "\n",
+    "By now you've successfully run your simple code generation and evaluation using Unify AI."
+   ]
+  }
+ ],
+ "metadata": {
+  "build_doc": {
+   "author": [
+    "riddhijivani122@gmail.com"
+   ],
+   "category": "local",
+   "section": "Flow",
+   "weight": 10
+  },
+  "description": "A quickstart tutorial to run a flex flow and evaluate it.",
+  "kernelspec": {
+   "display_name": "prompt_flow",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  },
+  "resources": "examples/requirements.txt, examples/flex-flows/basic"
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/flows/integrations/unify-ai/flow.flex.yaml

@@ -0,0 +1,7 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/flow.schema.json
+entry: programmer:write_simple_program
+environment:
+  python_requirements_txt: requirements.txt
+sample:
+  inputs:
+    text: Java Hello World!

examples/flows/integrations/unify-ai/hello.jinja2

@@ -0,0 +1,3 @@
+system: 
+Write a simple {{text}} program.
+Output code only.

examples/flows/integrations/unify-ai/llm.py

@@ -0,0 +1,47 @@
+import os
+
+from dotenv import load_dotenv
+from unify import Unify
+
+from promptflow.tracing import trace
+
+
+@trace
+def my_llm_tool(
+    prompt: str,
+    # for Unify AI, Model and Provider are to be specified by user.
+    model_name: str,
+    provider_name: str,
+    max_tokens: int = 1200,
+    temperature: float = 1.0,
+) -> str:
+    if "UNIFY_AI_API_KEY" not in os.environ:
+        # load environment variables from .env file
+        load_dotenv()
+
+    if "UNIFY_AI_API_KEY" not in os.environ:
+        raise Exception("Please specify environment variables: UNIFY_AI_API_KEY")
+    messages = [{"content": prompt, "role": "system"}]
+    api_key = os.environ.get("UNIFY_AI_API_KEY", None)
+    unify_client = Unify(
+        api_key=api_key,
+        model=model_name,
+        provider=provider_name,
+    )
+    response = unify_client.generate(
+        messages=messages,
+        max_tokens=int(max_tokens),
+        temperature=float(temperature),
+    )
+
+    # get first element because prompt is single.
+    return response
+
+
+if __name__ == "__main__":
+    result = my_llm_tool(
+        prompt="Write a simple Hello, world! program that displays the greeting message.",
+        model_name="llama-3.1-8b-chat",
+        provider_name="together-ai",
+    )
+    print(result)

examples/flows/integrations/unify-ai/programmer.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import TypedDict
+
+from jinja2 import Template
+from llm import my_llm_tool
+
+from promptflow.tracing import trace
+
+BASE_DIR = Path(__file__).absolute().parent
+
+
+class Result(TypedDict):
+    output: str
+
+
+@trace
+def load_prompt(jinja2_template: str, text: str) -> str:
+    """Load prompt function."""
+    with open(BASE_DIR / jinja2_template, "r", encoding="utf-8") as f:
+        prompt = Template(
+            f.read(), trim_blocks=True, keep_trailing_newline=True
+        ).render(text=text)
+        return prompt
+
+
+@trace
+def write_simple_program(
+    text: str = "Hello World!",
+    model_name="llama-3.1-8b-chat",
+    provider_name="together-ai",
+) -> Result:
+    """Ask LLM to write a simple program."""
+    prompt = load_prompt("hello.jinja2", text)
+    output = my_llm_tool(
+        prompt=prompt,
+        model_name=model_name,
+        provider_name=provider_name,
+        max_tokens=120,
+    )
+    return Result(output=output)
+
+
+if __name__ == "__main__":
+    from promptflow.tracing import start_trace
+
+    start_trace()
+    result = write_simple_program("Hello, world!")
+    print(result)

examples/flows/integrations/unify-ai/requirements.txt

@@ -0,0 +1,3 @@
+promptflow[azure]
+python-dotenv
+unifyai