chore: import upstream snapshot with attribution

examples/flex-flows/eval-checklist/README.md

@@ -0,0 +1,89 @@
+# Eval Check List
+A example flow defined using class entry which demos how to evaluate the answer pass user specified check list.
+
+## Prerequisites
+
+Install promptflow sdk and other dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+## Run flow
+
+- Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+
+- Setup connection
+
+Go to "Prompt flow" "Connections" tab. Click on "Create" button, select one of LLM tool supported connection types and fill in the configurations.
+
+Or use CLI to create connection:
+
+```bash
+# Override keys with --set to avoid yaml file changes
+pf connection create --file ../../connections/azure_openai.yml --set api_key=<your_api_key> api_base=<your_api_base> --name open_ai_connection
+```
+
+Note in [flow.flex.yaml](flow.flex.yaml) we are using connection named `open_ai_connection`.
+
+```bash
+# show registered connection
+pf connection show --name open_ai_connection
+```
+
+- Run as normal Python file
+
+```bash
+python check_list.py
+```
+
+- Test flow
+You'll need to write flow entry `flow.flex.yaml` to test with prompt flow.
+
+```bash
+pf flow test --flow . --init init.json --inputs sample.json
+```
+
+- Create run with multiple lines data
+
+```bash
+pf run create --flow . --init init.json --data ./data.jsonl --stream
+```
+
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.
+
+- List and show run meta
+
+```bash
+# list created run
+pf run list
+
+# get a sample run name
+
+name=$(pf run list -r 10 | jq '.[] | select(.name | contains("eval_checklist_")) | .name'| head -n 1 | tr -d '"')
+# show specific run detail
+pf run show --name $name
+
+# show output
+pf run show-details --name $name
+
+# visualize run in browser
+pf run visualize --name $name
+```
+
+## Run flow in cloud
+
+- Assume we already have a connection named `open_ai_connection` in workspace.
+
+```bash
+# set default workspace
+az account set -s <your_subscription_id>
+az configure --defaults group=<your_resource_group_name> workspace=<your_workspace_name>
+```
+
+- Create run
+
+```bash
+# run with environment variable reference connection in azureml workspace
+pfazure run create --flow . --init init.json --data ./data.jsonl --stream
+# run using yaml file
+pfazure run create --file run.yml --stream

examples/flex-flows/eval-checklist/check_list.py

@@ -0,0 +1,90 @@
+import json
+from pathlib import Path
+
+from promptflow.tracing import trace
+from promptflow.core import Prompty, AzureOpenAIModelConfiguration
+
+BASE_DIR = Path(__file__).absolute().parent
+
+
+@trace
+def check(answer: str, statement: str, model_config: AzureOpenAIModelConfiguration):
+    """Check the answer applies for the check statement."""
+    examples = [
+        {
+            "answer": "ChatGPT is a conversational AI model developed by OpenAI.",
+            "statement": "It contains a brief explanation of ChatGPT.",
+            "score": 5,
+            "explanation": "The statement is correct. The answer contains a brief explanation of ChatGPT.",
+        }
+    ]
+
+    prompty = Prompty.load(
+        source=BASE_DIR / "eval.prompty",
+        model={"configuration": model_config},
+    )
+    output = prompty(examples=examples, answer=answer, statement=statement)
+    output = json.loads(output)
+    return output
+
+
+class EvalFlow:
+    def __init__(self, model_config: AzureOpenAIModelConfiguration):
+        self.model_config = model_config
+
+    def __call__(self, answer: str, statements: dict):
+        """Check the answer applies for a collection of check statement."""
+        if isinstance(statements, str):
+            statements = json.loads(statements)
+
+        results = {}
+        for key, statement in statements.items():
+            r = check(
+                answer=answer, statement=statement, model_config=self.model_config
+            )
+            results[key] = r
+        return results
+
+    def __aggregate__(self, line_results: list) -> dict:
+        """Aggregate the results."""
+        total = len(line_results)
+        avg_correctness = (
+            sum(int(r["correctness"]["score"]) for r in line_results) / total
+        )
+        return {
+            "average_correctness": avg_correctness,
+            "total": total,
+        }
+
+
+if __name__ == "__main__":
+    from promptflow.tracing import start_trace
+
+    start_trace()
+
+    answer = """ChatGPT is a conversational AI model developed by OpenAI.
+    It is based on the GPT-3 architecture and is designed to generate human-like responses to text inputs.
+    ChatGPT is capable of understanding and responding to a wide range of topics and can be used for tasks such as
+    answering questions, generating creative content, and providing assistance with various tasks.
+    The model has been trained on a diverse range of internet text and is constantly being updated to improve its
+    performance and capabilities. ChatGPT is available through the OpenAI API and can be accessed by developers and
+    researchers to build applications and tools that leverage its capabilities."""
+    statements = {
+        "correctness": "It contains a detailed explanation of ChatGPT.",
+        "consise": "It is a consise statement.",
+    }
+
+    config = AzureOpenAIModelConfiguration(
+        connection="open_ai_connection", azure_deployment="gpt-4o"
+    )
+    flow = EvalFlow(config)
+
+    result = flow(
+        answer=answer,
+        statements=statements,
+    )
+    print(result)
+
+    # run aggregation
+    aggregation_result = flow.__aggregate__([result])
+    print(aggregation_result)

examples/flex-flows/eval-checklist/data.jsonl

@@ -0,0 +1 @@
+{"answer": "ChatGPT is a conversational AI model developed by OpenAI. It is based on the GPT-3 architecture and is designed to generate human-like responses to text inputs. ChatGPT is capable of understanding and responding to a wide range of topics and can be used for tasks such as answering questions, generating creative content, and providing assistance with various tasks. The model has been trained on a diverse range of internet text and is constantly being updated to improve its performance and capabilities. ChatGPT is available through the OpenAI API and can be accessed by developers and researchers to build applications and tools that leverage its capabilities.", "statements": { "correctness": "It contains a detailed explanation of ChatGPT." }}

examples/flex-flows/eval-checklist/eval.prompty

@@ -0,0 +1,43 @@
+
+---
+name: Evaluate based on a checklist
+description: Evaluate the quality of code snippet.
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: gpt-4o
+  parameters:
+    max_tokens: 256
+    temperature: 0.7
+
+inputs: 
+  examples:
+    type: list
+  answer:
+    type: string
+  statement:
+    type: string
+sample: ${file:sample.json}
+---
+
+# system:
+You are an AI assistant. 
+You task is to evaluate a score based on how the statement applies for the answer.
+Only accepts valid JSON format response without extra prefix or postfix.
+
+# user:
+This score value should always be an integer between 1 and 5. So the score produced should be 1 or 2 or 3 or 4 or 5.
+
+Here are a few examples:
+{% for ex in examples %}
+answer: {{ex.answer}}
+statement: {{ex.statement}}
+OUTPUT:
+{"score": "{{ex.score}}", "explanation":"{{ex.explanation}}"}
+{% endfor %}
+
+For a given answer, valuate the answer based on how the statement applies for the answer:
+answer: {{answer}}
+statement: {{statement}}
+OUTPUT:

examples/flex-flows/eval-checklist/flow.flex.yaml

@@ -0,0 +1,6 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+# flow is defined as python function
+entry: check_list:EvalFlow
+environment:
+  # image: mcr.microsoft.com/azureml/promptflow/promptflow-python
+  python_requirements_txt: requirements.txt

examples/flex-flows/eval-checklist/init.json

@@ -0,0 +1,6 @@
+{
+    "model_config": {
+        "connection": "open_ai_connection",
+        "azure_deployment": "gpt-4o"
+    }
+}

examples/flex-flows/eval-checklist/requirements.txt

@@ -0,0 +1 @@
+promptflow>=1.11.0

examples/flex-flows/eval-checklist/run.yml

@@ -0,0 +1,6 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json
+flow: .
+data: data.jsonl
+init:
+  connection: open_ai_connection
+

examples/flex-flows/eval-checklist/sample.json

@@ -0,0 +1,4 @@
+{
+    "answer": "ChatGPT is a conversational AI model developed by OpenAI. It is based on the GPT-3 architecture and is designed to generate human-like responses to text inputs. ChatGPT is capable of understanding and responding to a wide range of topics and can be used for tasks such as answering questions, generating creative content, and providing assistance with various tasks. The model has been trained on a diverse range of internet text and is constantly being updated to improve its performance and capabilities. ChatGPT is available through the OpenAI API and can be accessed by developers and researchers to build applications and tools that leverage its capabilities.",
+    "statements": { "correctness": "It contains a detailed explanation of ChatGPT." }
+}