chore: import upstream snapshot with attribution

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/aggregation.py

@@ -0,0 +1,24 @@
+from typing import Dict, List
+
+
+def aggregate(processed_results: List[int]) -> Dict[str, float]:
+    num_exception = 0
+    num_correct = 0
+
+    for i in range(len(processed_results)):
+        if processed_results[i] == -1:
+            num_exception += 1
+        elif processed_results[i] == 1:
+            num_correct += 1
+
+    num_total = len(processed_results)
+    accuracy = round(1.0 * num_correct / num_total, 2) if num_total else 0.0
+    error_rate = round(1.0 * num_exception / num_total, 2) if num_total else 0.0
+
+    return {
+        "num_total": num_total,
+        "num_correct": num_correct,
+        "num_exception": num_exception,
+        "accuracy": accuracy,
+        "error_rate": error_rate,
+    }

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/eval_runner.py

@@ -0,0 +1,70 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """Result of a batch evaluation run."""
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    """Runs a MAF workflow per row, collects outputs, then calls an aggregation function."""
+
+    def __init__(
+        self,
+        workflow_factory: Callable[[], Any],
+        aggregate_fn: Callable[..., dict],
+        concurrency: int = 5,
+        input_mapping: Optional[Dict[str, str]] = None,
+    ):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {
+                self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
+            }
+
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(
+            per_row_outputs=succeeded_outputs,
+            metrics=metrics,
+            errors=errors,
+        )
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/run_eval.py

@@ -0,0 +1,49 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "test_data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=str(obj["groundtruth"]), prediction=str(obj["prediction"])))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+    if result.errors:
+        print(f"\n--- Errors ({len(result.errors)}) ---")
+        for idx, err in result.errors:
+            print(f"  Row {idx}: {err}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/test_data.jsonl

@@ -0,0 +1,20 @@
+{"question": "What is the sum of 5 and 3?", "groundtruth": "8", "answer": "8"}
+{"question": "Subtract 7 from 10.", "groundtruth": "3", "answer": "3"}
+{"question": "Multiply 6 by 4.", "groundtruth": "24", "answer": "24"}
+{"question": "Divide 20 by 5.", "groundtruth": "4", "answer": "4"}
+{"question": "What is the square of 7?", "groundtruth": "49", "answer": "49"}
+{"question": "What is the square root of 81?", "groundtruth": "9", "answer": "9"}
+{"question": "If a rectangle has a length of 10 and width of 5, what is the area?", "groundtruth": "50", "answer": "50"}
+{"question": "A circle has a radius of 7, what is the area? (Use 3.14 for pi)", "groundtruth": "153.86", "answer": "153.871"}
+{"question": "Solve for x in the equation 2x + 3 = 9.", "groundtruth": "3", "answer": "3"}
+{"question": "What is the value of x if 5x = 25?", "groundtruth": "5", "answer": "5"}
+{"question": "A car travels 200 miles in 4 hours. What is the average speed of the car?", "groundtruth": "50", "answer": "50"}
+{"question": "A car travels at a speed of 60 mph. How long will it take to travel 180 miles?", "groundtruth": "3", "answer": "3"}
+{"question": "If a car travels at a speed of 40 mph for 2 hours, how far will it travel?","groundtruth": "80", "answer": "80"}
+{"question":"A rectangle has length = 10 cm and width = 5 cm. What is its area?", "groundtruth":"50", "answer": "50"}
+{"question":"A circle has radius = 7 cm. What is its circumference? (Use pi =3.14)", "groundtruth":"43.96", "answer": "43.959"}
+{"question":"A triangle has base =10 cm and height =5 cm. What is its area?", "groundtruth":"25", "answer": "25"}
+{"question":"What is the slope of the line that passes through (2,3) and (4,7)?", "groundtruth":"2", "answer": "2"}
+{"question":"The distance between A and B is 2000km, A is moving towards B with speed 80km/hour, meanwhile B is moving towards A with speed 120km/hour, how many hours later A and B can meet?", "groundtruth":"10", "answer": "10"}
+{"question":"The lengths of the two perpendicular sides of a right triangle are 6cm and 8cm. What is the length of the hypotenuse?", "groundtruth": "10", "answer": "10"}
+{"question":"A is running with average speed 10km/hour, A already run half hour. B start to chase A along the same route with average speed 15km/hour, how many hours B will take to meet A?", "groundtruth":"1", "answer": "2"}

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/test_eval.py

@@ -0,0 +1,75 @@
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_correct():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="1.0", prediction="1"))
+    assert result.get_outputs()[0] == 1
+    print("PASS: test_single_correct")
+
+
+async def test_single_incorrect():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="2.1", prediction="2.0"))
+    assert result.get_outputs()[0] == 0
+    print("PASS: test_single_incorrect")
+
+
+async def test_json_error():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="1.0", prediction="JSONDecodeError"))
+    assert result.get_outputs()[0] == -1
+    print("PASS: test_json_error")
+
+
+async def test_batch():
+    dataset = [
+        EvalInput(groundtruth="1.0", prediction="1"),
+        EvalInput(groundtruth="3.14", prediction="3.1415926"),
+        EvalInput(groundtruth="1.0", prediction="JSONDecodeError"),
+    ]
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+    assert result.metrics["num_total"] == 3
+    assert result.metrics["num_correct"] == 2
+    assert result.metrics["num_exception"] == 1
+    print("PASS: test_batch")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in test_data.jsonl"""
+    data_path = Path(__file__).parent / "test_data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["answer"],
+        ))
+        score = result.get_outputs()[0]
+        assert isinstance(score, int), f"Row {i}: expected int, got {type(score)}"
+        print(f"  Row {i}: score={score}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_correct()
+    await test_single_incorrect()
+    await test_json_error()
+    await test_batch()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/workflow.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class LineProcessExecutor(Executor):
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, int]) -> None:
+        processed_result = 0
+
+        if input.prediction == "JSONDecodeError" or input.prediction.startswith("Unknown Error:"):
+            await ctx.yield_output(-1)
+            return
+
+        try:
+            groundtruth = float(input.groundtruth)
+            prediction = float(input.prediction)
+        except ValueError:
+            await ctx.yield_output(-1)
+            return
+
+        if round(prediction, 2) == round(groundtruth, 2):
+            processed_result = 1
+
+        await ctx.yield_output(processed_result)
+
+
+def create_workflow():
+    _line_process = LineProcessExecutor(id="line_process")
+    return WorkflowBuilder(name="EvalAccuracyMathsRow", start_executor=_line_process).build()

examples/flows/evaluation/eval-accuracy-maths-to-code/aggregate.py

@@ -0,0 +1,37 @@
+from typing import List
+from promptflow.core import tool
+from promptflow.core import log_metric
+
+
+@tool
+def accuracy_aggregate(processed_results: List[int]):
+
+    num_exception = 0
+    num_correct = 0
+
+    for i in range(len(processed_results)):
+        if processed_results[i] == -1:
+            num_exception += 1
+        elif processed_results[i] == 1:
+            num_correct += 1
+
+    num_total = len(processed_results)
+    accuracy = round(1.0 * num_correct / num_total, 2)
+    error_rate = round(1.0 * num_exception / num_total, 2)
+
+    log_metric(key="accuracy", value=accuracy)
+    log_metric(key="error_rate", value=error_rate)
+
+    return {
+        "num_total": num_total,
+        "num_correct": num_correct,
+        "num_exception": num_exception,
+        "accuracy": accuracy,
+        "error_rate": error_rate
+    }
+
+
+if __name__ == "__main__":
+    numbers = [1, 1, 1, 1, 0, -1, -1]
+    accuracy = accuracy_aggregate(numbers)
+    print("The accuracy is", accuracy)

examples/flows/evaluation/eval-accuracy-maths-to-code/flow.dag.yaml

@@ -0,0 +1,29 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  groundtruth:
+    type: string
+    default: "1"
+  prediction:
+    type: string
+    default: "2"
+outputs:
+  score:
+    type: string
+    reference: ${line_process.output}
+nodes:
+- name: line_process
+  type: python
+  source:
+    type: code
+    path: line_process.py
+  inputs:
+    groundtruth: ${inputs.groundtruth}
+    prediction: ${inputs.prediction}
+- name: aggregate
+  type: python
+  source:
+    type: code
+    path: aggregate.py
+  inputs:
+    processed_results: ${line_process.output}
+  aggregation: true

examples/flows/evaluation/eval-accuracy-maths-to-code/line_process.py

@@ -0,0 +1,40 @@
+from promptflow.core import tool
+
+
+@tool
+def line_process(groundtruth: str, prediction: str) -> int:
+
+    processed_result = 0
+
+    if prediction == "JSONDecodeError" or prediction.startswith("Unknown Error:"):
+        processed_result = -1
+        return processed_result
+
+    try:
+        groundtruth = float(groundtruth)
+        prediction = float(prediction)
+    except ValueError:
+        processed_result = -1
+        return processed_result
+
+    if round(prediction, 2) == round(groundtruth, 2):
+        processed_result = 1
+
+    return processed_result
+
+
+if __name__ == "__main__":
+    processed_result = line_process("1.0", "1")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("3.14", "3.1415926")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("2.1", "2.0")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("1.0", "JSONDecodeError")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("1.0", "No module named 'numpy'")
+    print("The processed result is", processed_result)

examples/flows/evaluation/eval-accuracy-maths-to-code/test_data.jsonl

@@ -0,0 +1,20 @@
+{"question": "What is the sum of 5 and 3?", "groundtruth": "8", "answer": "8"}
+{"question": "Subtract 7 from 10.", "groundtruth": "3", "answer": "3"}
+{"question": "Multiply 6 by 4.", "groundtruth": "24", "answer": "24"}
+{"question": "Divide 20 by 5.", "groundtruth": "4", "answer": "4"}
+{"question": "What is the square of 7?", "groundtruth": "49", "answer": "49"}
+{"question": "What is the square root of 81?", "groundtruth": "9", "answer": "9"}
+{"question": "If a rectangle has a length of 10 and width of 5, what is the area?", "groundtruth": "50", "answer": "50"}
+{"question": "A circle has a radius of 7, what is the area? (Use 3.14 for pi)", "groundtruth": "153.86", "answer": "153.871"}
+{"question": "Solve for x in the equation 2x + 3 = 9.", "groundtruth": "3", "answer": "3"}
+{"question": "What is the value of x if 5x = 25?", "groundtruth": "5", "answer": "5"}
+{"question": "A car travels 200 miles in 4 hours. What is the average speed of the car?", "groundtruth": "50", "answer": "50"}
+{"question": "A car travels at a speed of 60 mph. How long will it take to travel 180 miles?", "groundtruth": "3", "answer": "3"}
+{"question": "If a car travels at a speed of 40 mph for 2 hours, how far will it travel?","groundtruth": "80", "answer": "80"}
+{"question":"A rectangle has length = 10 cm and width = 5 cm. What is its area?", "groundtruth":"50", "answer": "50"}
+{"question":"A circle has radius = 7 cm. What is its circumference? (Use pi =3.14)", "groundtruth":"43.96", "answer": "43.959"}
+{"question":"A triangle has base =10 cm and height =5 cm. What is its area?", "groundtruth":"25", "answer": "25"}
+{"question":"What is the slope of the line that passes through (2,3) and (4,7)?", "groundtruth":"2", "answer": "2"}
+{"question":"The distance between A and B is 2000km, A is moving towards B with speed 80km/hour, meanwhile B is moving towards A with speed 120km/hour, how many hours later A and B can meet?", "groundtruth":"10", "answer": "10"}
+{"question":"The lengths of the two perpendicular sides of a right triangle are 6cm and 8cm. What is the length of the hypotenuse?", "groundtruth": "10", "answer": "10"}
+{"question":"A is running with average speed 10km/hour, A already run half hour. B start to chase A along the same route with average speed 15km/hour, how many hours B will take to meet A?", "groundtruth":"1", "answer": "2"}

examples/flows/evaluation/eval-basic-maf/aggregation.py

@@ -0,0 +1,23 @@
+"""
+Aggregation function for eval-basic.
+
+Extracted from the original `aggregate.py` PromptFlow node.
+- `@tool` decorator removed (not needed in MAF).
+- `log_metric()` replaced with returning metrics as a dict.
+"""
+
+from typing import Dict, List
+
+
+def aggregate(processed_results: List[str]) -> Dict[str, int]:
+    """Aggregate per-row results into batch-level metrics.
+
+    :param processed_results: List of "Correct"/"Incorrect" strings from all rows.
+    :returns: Dict with metric name → value.
+    """
+    results_num = len(processed_results)
+    correct_num = processed_results.count("Correct")
+    return {
+        "results_num": results_num,
+        "correct_num": correct_num,
+    }

examples/flows/evaluation/eval-basic-maf/data.jsonl

@@ -0,0 +1 @@
+{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow."}

examples/flows/evaluation/eval-basic-maf/eval_runner.py

@@ -0,0 +1,116 @@
+"""
+EvalRunner — batch evaluation orchestrator for MAF workflows.
+
+Bridges the gap between MAF's single-invocation workflow model and PromptFlow's
+batch-level `aggregation: true` pattern.
+
+Usage:
+    runner = EvalRunner(workflow, aggregate_fn, input_mapping={"values": "processed_results"})
+    result = await runner.run(dataset)
+    print(result.metrics)
+"""
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """Result of a batch evaluation run."""
+
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    """Runs a MAF workflow per row, collects outputs, then calls an aggregation function.
+
+    This mirrors PromptFlow's two-phase execution model:
+      Phase 1 — run each row through the workflow concurrently
+      Phase 2 — pass all collected outputs to the aggregation function
+
+    MAF workflows do not support concurrent execution on a single instance,
+    so `workflow_factory` creates a fresh workflow for each concurrent row.
+
+    :param workflow_factory: A zero-arg callable that returns a built MAF workflow.
+    :param aggregate_fn: A function that receives collected outputs and returns a metrics dict.
+    :param concurrency: Max concurrent workflow.run() calls (prevents rate-limit errors).
+    :param input_mapping: Optional rename map for transposed keys → aggregation function params.
+        For single-value outputs, _transpose produces {"values": [...]}.  If the aggregation
+        function expects a different param name (e.g., "processed_results"), pass
+        {"values": "processed_results"}.
+    """
+
+    def __init__(
+        self,
+        workflow_factory: Callable[[], Any],
+        aggregate_fn: Callable[..., dict],
+        concurrency: int = 5,
+        input_mapping: Optional[Dict[str, str]] = None,
+    ):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        """Execute the full eval pipeline: per-row → collect → aggregate.
+
+        :param dataset: List of inputs to pass to workflow.run() (one per row).
+        :returns: EvalResult with per-row outputs, metrics, and any errors.
+        """
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        # Phase 1: run all rows concurrently (bounded by semaphore)
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Separate successes from failures
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        # Transpose outputs into aggregation inputs
+        aggregation_inputs = self._transpose(succeeded_outputs)
+
+        # Apply parameter name mapping if provided
+        if self._input_mapping:
+            aggregation_inputs = {
+                self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
+            }
+
+        # Phase 2: call aggregation function
+        metrics = self._aggregate_fn(**aggregation_inputs)
+
+        return EvalResult(
+            per_row_outputs=succeeded_outputs,
+            metrics=metrics,
+            errors=errors,
+        )
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        """Transpose per-row outputs into aggregation-ready keyword args.
+
+        - If outputs are plain values (str, int, float): {"values": [v1, v2, ...]}
+        - If outputs are dicts: {key: [row1[key], row2[key], ...]} for each key
+        """
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-basic-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-basic-maf/run_eval.py

@@ -0,0 +1,70 @@
+"""
+Entry point: run the eval-basic evaluation as a batch.
+
+Loads the test dataset, runs the per-row workflow for each row,
+then aggregates results.
+
+Usage:
+    python run_eval.py
+    python run_eval.py --data path/to/data.jsonl --concurrency 10
+"""
+
+import argparse
+import asyncio
+import json
+from pathlib import Path
+
+from aggregation import aggregate
+from eval_runner import EvalResult, EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    """Load a JSONL file into a list of EvalInput."""
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=obj["groundtruth"], prediction=obj["prediction"]))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int) -> EvalResult:
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "processed_results"},
+    )
+
+    result = await runner.run(dataset)
+
+    print("\n--- Per-row outputs ---")
+    for i, output in enumerate(result.per_row_outputs):
+        print(f"  Row {i}: {output}")
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+    if result.errors:
+        print(f"\n--- Errors ({len(result.errors)}) ---")
+        for idx, err in result.errors:
+            print(f"  Row {idx}: {err}")
+
+    return result
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run eval-basic evaluation batch")
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA, help="Path to JSONL dataset")
+    parser.add_argument("--concurrency", type=int, default=5, help="Max concurrent workflow runs")
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-basic-maf/test_eval_basic.py

@@ -0,0 +1,100 @@
+"""
+Test script for the eval-basic MAF conversion.
+
+Verifies both the per-row workflow and the full batch evaluation pipeline.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Test the per-row workflow with a matching pair."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="Sunny"))
+    output = result.get_outputs()[0]
+    assert output == "Correct", f"Expected 'Correct', got '{output}'"
+    print("PASS: test_single_row")
+
+
+async def test_single_row_mismatch():
+    """Test the per-row workflow with a non-matching pair."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="rainy"))
+    output = result.get_outputs()[0]
+    assert output == "Incorrect", f"Expected 'Incorrect', got '{output}'"
+    print("PASS: test_single_row_mismatch")
+
+
+async def test_batch_evaluation():
+    """Test the full batch pipeline with EvalRunner."""
+    dataset = [
+        EvalInput(groundtruth="APP", prediction="APP"),
+        EvalInput(groundtruth="APP", prediction="WEB"),
+        EvalInput(groundtruth="DB", prediction="db"),
+    ]
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+
+    assert result.per_row_outputs == ["Correct", "Incorrect", "Correct"], (
+        f"Unexpected per-row outputs: {result.per_row_outputs}"
+    )
+    assert result.metrics["results_num"] == 3, f"Expected 3 results, got {result.metrics['results_num']}"
+    assert result.metrics["correct_num"] == 2, f"Expected 2 correct, got {result.metrics['correct_num']}"
+    assert len(result.errors) == 0, f"Unexpected errors: {result.errors}"
+    print("PASS: test_batch_evaluation")
+
+
+async def test_empty_dataset():
+    """Test with an empty dataset."""
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run([])
+    assert result.per_row_outputs == []
+    assert result.metrics["results_num"] == 0
+    assert len(result.errors) == 0
+    print("PASS: test_empty_dataset")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["prediction"],
+        ))
+        grade = result.get_outputs()[0]
+        assert grade in ("Correct", "Incorrect"), f"Row {i}: unexpected grade '{grade}'"
+        print(f"  Row {i}: grade={grade}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_single_row_mismatch()
+    await test_batch_evaluation()
+    await test_empty_dataset()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-basic-maf/workflow.py

@@ -0,0 +1,56 @@
+"""
+Per-row MAF workflow for eval-basic.
+
+Converts the `line_process` node from the original PromptFlow evaluation flow.
+Each workflow invocation processes a single (groundtruth, prediction) pair and
+yields "Correct" or "Incorrect".
+
+Original flow graph (per-row nodes only):
+    [inputs: groundtruth, prediction] → [line_process] → output
+"""
+
+import asyncio
+from dataclasses import dataclass
+
+from typing_extensions import Never
+
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class LineProcessExecutor(Executor):
+    """Replaces the `line_process` Python node.
+
+    Compares groundtruth and prediction (case-insensitive) and yields the result.
+    """
+
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, str]) -> None:
+        result = "Correct" if input.groundtruth.lower() == input.prediction.lower() else "Incorrect"
+        await ctx.yield_output(result)
+
+
+def create_workflow():
+    """Create a fresh workflow instance.
+
+    MAF workflows do not support concurrent execution, so each batch row
+    needs its own workflow instance.
+    """
+    _line_process = LineProcessExecutor(id="line_process")
+    return WorkflowBuilder(name="EvalBasicRow", start_executor=_line_process).build()
+
+
+async def main():
+    """Quick smoke test with a single row."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="Sunny"))
+    print(result.get_outputs()[0])  # → "Correct"
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-basic/README.md

@@ -0,0 +1,43 @@
+# Basic Eval
+This example shows how to create a basic evaluation flow. 
+
+Tools used in this flow：
+- `python` tool
+
+## Prerequisites
+
+Install promptflow sdk and other dependencies in this folder:
+```bash
+pip install -r requirements.txt
+```
+
+## What you will learn
+
+In this flow, you will learn
+- how to compose a point based evaluation flow, where you can calculate point-wise metrics.
+- the way to log metrics. use `from promptflow.core import log_metric`
+    - see file [aggregate](aggregate.py).
+
+### 1. Test flow with single line data
+
+Testing flow/node:
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+
+# test with flow inputs
+pf flow test --flow . --inputs groundtruth=ABC prediction=ABC
+
+# test node with inputs
+pf flow test --flow . --node line_process --inputs groundtruth=ABC prediction=ABC
+```
+
+### 2. create flow run with multi line data
+There are two ways to evaluate an classification flow.
+
+```bash
+pf run create --flow . --data ./data.jsonl --column-mapping groundtruth='${data.groundtruth}' prediction='${data.prediction}' --stream
+```
+
+You can also skip providing `column-mapping` if provided data has same column name as the flow.
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.

examples/flows/evaluation/eval-basic/aggregate.py

@@ -0,0 +1,24 @@
+from typing import List
+
+from promptflow.core import tool
+
+
+@tool
+def aggregate(processed_results: List[str]):
+    """
+    This tool aggregates the processed result of all lines to the variant level and log metric for each variant.
+
+    :param processed_results: List of the output of line_process node.
+    """
+
+    # Add your aggregation logic here
+    # aggregated_results should be a dictionary with the metric name as the key and the metric value as the value.
+    results_num = len(processed_results)
+    print(results_num)
+    print(processed_results)
+
+    # Log metric for each variant
+    from promptflow.core import log_metric
+    log_metric(key="results_num", value=results_num)
+
+    return results_num

examples/flows/evaluation/eval-basic/data.jsonl

@@ -0,0 +1 @@
+{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow."}

examples/flows/evaluation/eval-basic/flow.dag.yaml

@@ -0,0 +1,31 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  groundtruth:
+    type: string
+    default: groundtruth
+  prediction:
+    type: string
+    default: prediction
+outputs:
+  results:
+    type: string
+    reference: ${line_process.output}
+nodes:
+- name: line_process
+  type: python
+  source:
+    type: code
+    path: line_process.py
+  inputs:
+    groundtruth: ${inputs.groundtruth}
+    prediction: ${inputs.prediction}
+- name: aggregate
+  type: python
+  source:
+    type: code
+    path: aggregate.py
+  inputs:
+    processed_results: ${line_process.output}
+  aggregation: true
+environment:
+    python_requirements_txt: requirements.txt

examples/flows/evaluation/eval-basic/line_process.py

@@ -0,0 +1,14 @@
+from promptflow.core import tool
+
+
+@tool
+def line_process(groundtruth: str, prediction: str):
+    """
+    This tool processes the prediction of a single line and returns the processed result.
+
+    :param groundtruth: the groundtruth of a single line.
+    :param prediction: the prediction of a single line.
+    """
+
+    # Add your line processing logic here
+    return "Correct" if groundtruth.lower() == prediction.lower() else "Incorrect"

examples/flows/evaluation/eval-basic/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

examples/flows/evaluation/eval-chat-math-maf/aggregation.py

@@ -0,0 +1,24 @@
+from typing import Dict, List
+
+
+def aggregate(processed_results: List[int]) -> Dict[str, float]:
+    num_exception = 0
+    num_correct = 0
+
+    for i in range(len(processed_results)):
+        if processed_results[i] == -1:
+            num_exception += 1
+        elif processed_results[i] == 1:
+            num_correct += 1
+
+    num_total = len(processed_results)
+    accuracy = round(1.0 * num_correct / num_total, 2) if num_total else 0.0
+    error_rate = round(1.0 * num_exception / num_total, 2) if num_total else 0.0
+
+    return {
+        "num_total": num_total,
+        "num_correct": num_correct,
+        "num_exception": num_exception,
+        "accuracy": accuracy,
+        "error_rate": error_rate,
+    }

examples/flows/evaluation/eval-chat-math-maf/data.jsonl

@@ -0,0 +1,3 @@
+{"groundtruth": "10","prediction": "10"}
+{"groundtruth": "253","prediction": "506"}
+{"groundtruth": "1/3","prediction": "2/6"}

examples/flows/evaluation/eval-chat-math-maf/eval_runner.py

@@ -0,0 +1,70 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """Result of a batch evaluation run."""
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    """Runs a MAF workflow per row, collects outputs, then calls an aggregation function."""
+
+    def __init__(
+        self,
+        workflow_factory: Callable[[], Any],
+        aggregate_fn: Callable[..., dict],
+        concurrency: int = 5,
+        input_mapping: Optional[Dict[str, str]] = None,
+    ):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {
+                self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
+            }
+
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(
+            per_row_outputs=succeeded_outputs,
+            metrics=metrics,
+            errors=errors,
+        )
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-chat-math-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-chat-math-maf/run_eval.py

@@ -0,0 +1,45 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=str(obj["groundtruth"]), prediction=str(obj["prediction"])))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-chat-math-maf/test_eval.py

@@ -0,0 +1,74 @@
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_fraction_match():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="3/5", prediction="6/10"))
+    assert result.get_outputs()[0] == 1
+    print("PASS: test_fraction_match")
+
+
+async def test_float_fraction():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="1/2", prediction="0.5"))
+    assert result.get_outputs()[0] == 1
+    print("PASS: test_float_fraction")
+
+
+async def test_mismatch():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="3", prediction="5"))
+    assert result.get_outputs()[0] == -1
+    print("PASS: test_mismatch")
+
+
+async def test_batch():
+    dataset = [
+        EvalInput(groundtruth="3/5", prediction="6/10"),
+        EvalInput(groundtruth="1/2", prediction="0.5"),
+        EvalInput(groundtruth="3", prediction="5"),
+    ]
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+    assert result.metrics["num_correct"] == 2
+    assert result.metrics["num_exception"] == 1
+    print("PASS: test_batch")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["prediction"],
+        ))
+        score = result.get_outputs()[0]
+        assert isinstance(score, int), f"Row {i}: expected int, got {type(score)}"
+        print(f"  Row {i}: score={score}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_fraction_match()
+    await test_float_fraction()
+    await test_mismatch()
+    await test_batch()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-chat-math-maf/workflow.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+def string_to_number(raw_string: str) -> float:
+    float_number = 0.0
+    try:
+        float_number = float(raw_string)
+    except Exception:
+        if '/' in raw_string:
+            split_list = raw_string.split('/')
+            if len(split_list) == 2:
+                numerator, denominator = split_list
+                try:
+                    float_number = float(numerator) / float(denominator)
+                except Exception:
+                    return None
+            else:
+                return None
+        else:
+            return None
+    return float_number
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class LineProcessExecutor(Executor):
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, int]) -> None:
+        pred_float = string_to_number(input.prediction)
+        if pred_float is None:
+            await ctx.yield_output(-1)
+            return
+        gt_float = string_to_number(input.groundtruth)
+        if gt_float is None:
+            await ctx.yield_output(-1)
+            return
+        if round(pred_float, 10) == round(gt_float, 10):
+            await ctx.yield_output(1)
+        else:
+            await ctx.yield_output(-1)
+
+
+def create_workflow():
+    _line_process = LineProcessExecutor(id="line_process")
+    return WorkflowBuilder(name="EvalChatMathRow", start_executor=_line_process).build()

examples/flows/evaluation/eval-chat-math/README.md

@@ -0,0 +1,36 @@
+# Eval chat math
+
+This example shows how to evaluate the answer of math questions, which can compare the output results with the standard answers numerically.
+
+Learn more on corresponding [tutorials](../../../tutorials/flow-fine-tuning-evaluation/promptflow-quality-improvement.md)
+
+Tools used in this flow：
+- `python` tool
+
+## Prerequisites
+
+Install promptflow sdk and other dependencies in this folder:
+```bash
+pip install -r requirements.txt
+```
+
+### 1. Test flow with single line data
+
+Testing flow/node:
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+
+# test with flow inputs
+pf flow test --flow . --inputs groundtruth=123 prediction=123
+
+# test node with inputs
+pf flow test --flow . --node line_process --inputs groundtruth=123 prediction=123
+```
+
+### 2. create flow run with multi line data
+There are two ways to evaluate an classification flow.
+
+```bash
+pf run create --flow . --data ./data.jsonl --stream
+```

examples/flows/evaluation/eval-chat-math/aggregate.py

@@ -0,0 +1,37 @@
+from typing import List
+from promptflow.core import tool
+from promptflow.core import log_metric
+
+
+@tool
+def accuracy_aggregate(processed_results: List[int]):
+
+    num_exception = 0
+    num_correct = 0
+
+    for i in range(len(processed_results)):
+        if processed_results[i] == -1:
+            num_exception += 1
+        elif processed_results[i] == 1:
+            num_correct += 1
+
+    num_total = len(processed_results)
+    accuracy = round(1.0 * num_correct / num_total, 2)
+    error_rate = round(1.0 * num_exception / num_total, 2)
+
+    log_metric(key="accuracy", value=accuracy)
+    log_metric(key="error_rate", value=error_rate)
+
+    return {
+        "num_total": num_total,
+        "num_correct": num_correct,
+        "num_exception": num_exception,
+        "accuracy": accuracy,
+        "error_rate": error_rate
+    }
+
+
+if __name__ == "__main__":
+    numbers = [1, 1, 1, 1, 0, -1, -1]
+    accuracy = accuracy_aggregate(numbers)
+    print("The accuracy is", accuracy)

examples/flows/evaluation/eval-chat-math/data.jsonl

@@ -0,0 +1,3 @@
+{"groundtruth": "10","prediction": "10"}
+{"groundtruth": "253","prediction": "506"}
+{"groundtruth": "1/3","prediction": "2/6"}

examples/flows/evaluation/eval-chat-math/flow.dag.yaml

@@ -0,0 +1,36 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  groundtruth:
+    type: string
+    default: "10"
+    is_chat_input: false
+  prediction:
+    type: string
+    default: "10"
+    is_chat_input: false
+outputs:
+  score:
+    type: string
+    reference: ${line_process.output}
+nodes:
+- name: line_process
+  type: python
+  source:
+    type: code
+    path: line_process.py
+  inputs:
+    groundtruth: ${inputs.groundtruth}
+    prediction: ${inputs.prediction}
+  use_variants: false
+- name: aggregate
+  type: python
+  source:
+    type: code
+    path: aggregate.py
+  inputs:
+    processed_results: ${line_process.output}
+  aggregation: true
+  use_variants: false
+node_variants: {}
+environment:
+  python_requirements_txt: requirements.txt

examples/flows/evaluation/eval-chat-math/line_process.py

@@ -0,0 +1,55 @@
+from promptflow.core import tool
+
+
+def string_to_number(raw_string: str) -> float:
+    ''' Try to parse the prediction string and groundtruth string to float number.
+    Support parse int, float, fraction and recognize non-numeric string with wrong format.
+    Wrong format cases: 'the answer is \box{2/3}', '0, 5, or any number greater than 11', '4/7//9'
+    '''
+    float_number = 0.0
+    try:
+        float_number = float(raw_string)
+    except Exception:
+        if '/' in raw_string:
+            split_list = raw_string.split('/')
+            if len(split_list) == 2:
+                numerator, denominator = split_list
+                try:
+                    float_number = float(numerator) / float(denominator)
+                except Exception:
+                    return None
+            else:
+                return None
+        else:
+            return None
+    return float_number
+
+
+@tool
+def line_process(groundtruth: str, prediction: str) -> int:
+    pred_float = string_to_number(prediction)
+    '''Early stop'''
+    if (pred_float is None):
+        return -1
+    gt_float = string_to_number(groundtruth)
+    if (gt_float is None):
+        return -1
+    ''' both pred_float and gt_float are valid'''
+    if round(pred_float, 10) == round(gt_float, 10):
+        return 1
+    else:
+        return -1
+
+
+if __name__ == "__main__":
+    processed_result = line_process("3/5", "6/10")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("1/2", "0.5")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("3", "5")
+    print("The processed result is", processed_result)
+
+    processed_result = line_process("2/3", "the answer is \box{2/3}")
+    print("The processed result is", processed_result)

examples/flows/evaluation/eval-chat-math/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

examples/flows/evaluation/eval-classification-accuracy-maf/aggregation.py

@@ -0,0 +1,6 @@
+from typing import Dict, List
+
+
+def aggregate(grades: List[str]) -> Dict[str, float]:
+    accuracy = round((grades.count("Correct") / len(grades)), 2) if grades else 0.0
+    return {"accuracy": accuracy}

examples/flows/evaluation/eval-classification-accuracy-maf/data.jsonl

@@ -0,0 +1,3 @@
+{"groundtruth": "App","prediction": "App"}
+{"groundtruth": "Channel","prediction": "Channel"}
+{"groundtruth": "Academic","prediction": "Academic"}

examples/flows/evaluation/eval-classification-accuracy-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-classification-accuracy-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-classification-accuracy-maf/run_eval.py

@@ -0,0 +1,45 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=obj["groundtruth"], prediction=obj["prediction"]))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "grades"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-classification-accuracy-maf/test_eval.py

@@ -0,0 +1,65 @@
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_correct():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="APP", prediction="APP"))
+    assert result.get_outputs()[0] == "Correct"
+    print("PASS: test_correct")
+
+
+async def test_incorrect():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="APP", prediction="WEB"))
+    assert result.get_outputs()[0] == "Incorrect"
+    print("PASS: test_incorrect")
+
+
+async def test_batch():
+    dataset = [
+        EvalInput(groundtruth="APP", prediction="APP"),
+        EvalInput(groundtruth="Channel", prediction="Channel"),
+        EvalInput(groundtruth="Academic", prediction="Finance"),
+    ]
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "grades"},
+    )
+    result = await runner.run(dataset)
+    assert result.metrics["accuracy"] == 0.67
+    print("PASS: test_batch")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["prediction"],
+        ))
+        grade = result.get_outputs()[0]
+        assert grade in ("Correct", "Incorrect"), f"Row {i}: unexpected grade '{grade}'"
+        print(f"  Row {i}: grade={grade}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_correct()
+    await test_incorrect()
+    await test_batch()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-classification-accuracy-maf/workflow.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class GradeExecutor(Executor):
+    @handler
+    async def grade(self, input: EvalInput, ctx: WorkflowContext[Never, str]) -> None:
+        result = "Correct" if input.groundtruth.lower() == input.prediction.lower() else "Incorrect"
+        await ctx.yield_output(result)
+
+
+def create_workflow():
+    _grade = GradeExecutor(id="grade")
+    return WorkflowBuilder(name="EvalClassificationAccuracyRow", start_executor=_grade).build()

examples/flows/evaluation/eval-classification-accuracy/README.md

@@ -0,0 +1,50 @@
+# Classification Accuracy Evaluation
+
+This is a flow illustrating how to evaluate the performance of a classification system. It involves comparing each prediction to the groundtruth and assigns a "Correct" or "Incorrect" grade, and aggregating the results to produce metrics such as accuracy, which reflects how good the system is at classifying the data.
+
+Tools used in this flow：
+- `python` tool
+
+## What you will learn
+
+In this flow, you will learn
+- how to compose a point based evaluation flow, where you can calculate point-wise metrics.
+- the way to log metrics. use `from promptflow.core import log_metric`
+    - see file [calculate_accuracy.py](calculate_accuracy.py)
+
+### 0. Setup connection
+
+Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+
+```bash
+# Override keys with --set to avoid yaml file changes
+pf connection create --file ../../../connections/azure_openai.yml --set api_key=<your_api_key> api_base=<your_api_base>
+```
+
+### 1. Test flow/node
+
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+
+# test with flow inputs
+pf flow test --flow . --inputs groundtruth=APP prediction=APP
+
+# test node with inputs
+pf flow test --flow . --node grade --inputs groundtruth=groundtruth prediction=prediction
+```
+
+### 2. create flow run with multi line data
+There are two ways to evaluate an classification flow.
+
+```bash
+pf run create --flow . --data ./data.jsonl --column-mapping groundtruth='${data.groundtruth}' prediction='${data.prediction}' --stream
+```
+
+You can also skip providing `column-mapping` if provided data has same column name as the flow.
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.
+
+### 3. create run against other flow run
+
+Learn more in [web-classification](../../standard/web-classification/README.md)
+

examples/flows/evaluation/eval-classification-accuracy/calculate_accuracy.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from promptflow.core import log_metric, tool
+
+
+@tool
+def calculate_accuracy(grades: List[str]):
+    result = []
+    for index in range(len(grades)):
+        grade = grades[index]
+        result.append(grade)
+
+    # calculate accuracy for each variant
+    accuracy = round((result.count("Correct") / len(result)), 2)
+    log_metric("accuracy", accuracy)
+
+    return result

examples/flows/evaluation/eval-classification-accuracy/data.jsonl

@@ -0,0 +1,3 @@
+{"groundtruth": "App","prediction": "App"}
+{"groundtruth": "Channel","prediction": "Channel"}
+{"groundtruth": "Academic","prediction": "Academic"}

examples/flows/evaluation/eval-classification-accuracy/flow.dag.yaml

@@ -0,0 +1,35 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  groundtruth:
+    type: string
+    description: Please specify the groundtruth column, which contains the true label
+      to the outputs that your flow produces.
+    default: APP
+  prediction:
+    type: string
+    description: Please specify the prediction column, which contains the predicted
+      outputs that your flow produces.
+    default: APP
+outputs:
+  grade:
+    type: string
+    reference: ${grade.output}
+nodes:
+- name: grade
+  type: python
+  source:
+    type: code
+    path: grade.py
+  inputs:
+    groundtruth: ${inputs.groundtruth}
+    prediction: ${inputs.prediction}
+- name: calculate_accuracy
+  type: python
+  source:
+    type: code
+    path: calculate_accuracy.py
+  inputs:
+    grades: ${grade.output}
+  aggregation: true
+environment:
+    python_requirements_txt: requirements.txt

examples/flows/evaluation/eval-classification-accuracy/grade.py

@@ -0,0 +1,6 @@
+from promptflow.core import tool
+
+
+@tool
+def grade(groundtruth: str, prediction: str):
+    return "Correct" if groundtruth.lower() == prediction.lower() else "Incorrect"

examples/flows/evaluation/eval-classification-accuracy/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

examples/flows/evaluation/eval-entity-match-rate-maf/aggregation.py

@@ -0,0 +1,12 @@
+from typing import Dict, List
+
+
+def aggregate(exact_match: List[int], partial_match: List[int],
+              answer: List[list], ground_truth: List[list]) -> Dict[str, float]:
+    n = len(exact_match)
+    exact_match_rate = sum(exact_match) / n if n else 0.0
+    partial_match_rate = sum(partial_match) / n if n else 0.0
+    return {
+        "exact_match_rate": exact_match_rate,
+        "partial_match_rate": partial_match_rate,
+    }

examples/flows/evaluation/eval-entity-match-rate-maf/data.jsonl

@@ -0,0 +1,2 @@
+{"entities": ["software engineer","CEO"],"ground_truth": "\"CEO, Software Engineer, Finance Manager\""}
+{"entities": ["Software Engineer","CEO", "Finance Manager"],"ground_truth": "\"CEO, Software Engineer, Finance Manager\""}

examples/flows/evaluation/eval-entity-match-rate-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-entity-match-rate-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-entity-match-rate-maf/run_eval.py

@@ -0,0 +1,44 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(entities=obj["entities"], ground_truth=obj["ground_truth"]))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-entity-match-rate-maf/test_eval.py

@@ -0,0 +1,60 @@
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_exact_match():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(
+        entities=["software engineer", "CEO"],
+        ground_truth='"CEO, Software Engineer, Finance Manager"'
+    ))
+    output = result.get_outputs()[0]
+    assert output["exact_match"] == 0
+    assert output["partial_match"] == 1
+    print("PASS: test_exact_match")
+
+
+async def test_batch():
+    dataset = [
+        EvalInput(entities=["CEO"], ground_truth="CEO, Engineer"),
+        EvalInput(entities=["CEO", "Engineer"], ground_truth="CEO, Engineer"),
+    ]
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+    )
+    result = await runner.run(dataset)
+    assert result.metrics["partial_match_rate"] == 1.0
+    print("PASS: test_batch")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            entities=row["entities"],
+            ground_truth=row["ground_truth"],
+        ))
+        output = result.get_outputs()[0]
+        assert isinstance(output, dict), f"Row {i}: expected dict, got {type(output)}"
+        print(f"  Row {i}: output={output}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_exact_match()
+    await test_batch()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-entity-match-rate-maf/workflow.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from typing import List
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    entities: List[str]
+    ground_truth: str
+
+
+def cleansing(entities_str: str) -> List[str]:
+    parts = entities_str.split(",")
+    cleaned_parts = [part.strip(" \t.\"") for part in parts]
+    return [part for part in cleaned_parts if len(part) > 0]
+
+
+def is_match(answer: List[str], ground_truth: List[str],
+             ignore_case: bool, ignore_order: bool, allow_partial: bool) -> bool:
+    if ignore_case:
+        answer = [a.lower() for a in answer]
+        ground_truth = [g.lower() for g in ground_truth]
+    if ignore_order:
+        answer.sort()
+        ground_truth.sort()
+    if allow_partial:
+        x = [a for a in answer if a in ground_truth]
+        return x == answer
+    return answer == ground_truth
+
+
+class EntityMatchExecutor(Executor):
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, dict]) -> None:
+        ground_truth_list = cleansing(input.ground_truth)
+
+        exact_match = 0
+        partial_match = 0
+        if is_match(input.entities, ground_truth_list, ignore_case=True, ignore_order=True, allow_partial=False):
+            exact_match = 1
+        if is_match(input.entities, ground_truth_list, ignore_case=True, ignore_order=True, allow_partial=True):
+            partial_match = 1
+
+        await ctx.yield_output({
+            "exact_match": exact_match,
+            "partial_match": partial_match,
+            "answer": input.entities,
+            "ground_truth": ground_truth_list,
+        })
+
+
+def create_workflow():
+    _matcher = EntityMatchExecutor(id="entity_match")
+    return WorkflowBuilder(name="EvalEntityMatchRow", start_executor=_matcher).build()

examples/flows/evaluation/eval-entity-match-rate/README.md

@@ -0,0 +1,29 @@
+# Entity match rate evaluation
+
+This is a flow evaluates: entity match rate.
+
+Tools used in this flow：
+- `python` tool
+
+## Prerequisites
+
+Install promptflow sdk and other dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+### 1. Test flow/node
+
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+```
+
+### 2. create flow run with multi line data
+
+```bash
+pf run create --flow . --data ./data.jsonl --column-mapping ground_truth='${data.ground_truth}' entities='${data.entities}' --stream
+```
+
+You can also skip providing `column-mapping` if provided data has same column name as the flow.
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.

examples/flows/evaluation/eval-entity-match-rate/cleansing.py

@@ -0,0 +1,11 @@
+from typing import List
+from promptflow.core import tool
+
+
+@tool
+def cleansing(entities_str: str) -> List[str]:
+    # Split, remove leading and trailing spaces/tabs/dots
+    parts = entities_str.split(",")
+    cleaned_parts = [part.strip(" \t.\"") for part in parts]
+    entities = [part for part in cleaned_parts if len(part) > 0]
+    return entities

examples/flows/evaluation/eval-entity-match-rate/data.jsonl

@@ -0,0 +1,2 @@
+{"entities": ["software engineer","CEO"],"ground_truth": "\"CEO, Software Engineer, Finance Manager\""}
+{"entities": ["Software Engineer","CEO", "Finance Manager"],"ground_truth": "\"CEO, Software Engineer, Finance Manager\""}

examples/flows/evaluation/eval-entity-match-rate/flow.dag.yaml

@@ -0,0 +1,40 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+inputs:
+  entities:
+    type: list
+    default:
+    - software engineer
+    - CEO
+  ground_truth:
+    type: string
+    default: '"CEO, Software Engineer, Finance Manager"'
+outputs:
+  match_cnt:
+    type: object
+    reference: ${match.output}
+nodes:
+- name: cleansing
+  type: python
+  source:
+    type: code
+    path: cleansing.py
+  inputs:
+    entities_str: ${inputs.ground_truth}
+- name: match
+  type: python
+  source:
+    type: code
+    path: match.py
+  inputs:
+    answer: ${inputs.entities}
+    ground_truth: ${cleansing.output}
+- name: log_metrics
+  type: python
+  source:
+    type: code
+    path: log_metrics.py
+  inputs:
+    match_counts: ${match.output}
+  aggregation: true
+environment:
+  python_requirements_txt: requirements.txt

examples/flows/evaluation/eval-entity-match-rate/is_match_test.py

@@ -0,0 +1,13 @@
+import unittest
+
+from match import is_match
+
+
+class IsMatchTest(unittest.TestCase):
+    def test_normal(self):
+        self.assertEqual(is_match(["a", "b"], ["B", "a"], True, True, False), True)
+        self.assertEqual(is_match(["a", "b"], ["B", "a"], True, False, False), False)
+        self.assertEqual(is_match(["a", "b"], ["B", "a"], False, True, False), False)
+        self.assertEqual(is_match(["a", "b"], ["B", "a"], False, False, True), False)
+        self.assertEqual(is_match(["a", "b"], ["a", "b"], False, False, False), True)
+        self.assertEqual(is_match(["a", "b"], ["a", "b", "c"], True, False, True), True)

examples/flows/evaluation/eval-entity-match-rate/log_metrics.py

@@ -0,0 +1,20 @@
+from promptflow.core import tool
+from typing import List
+from promptflow.core import log_metric
+
+# The inputs section will change based on the arguments of the tool function, after you save the code
+# Adding type to arguments and return value will help the system show the types properly
+# Please update the function name/signature per need
+
+
+@tool
+def log_metrics(match_counts: List[dict]):
+    exact_match_rate = sum([m["exact_match"] for m in match_counts]) / len(match_counts)
+    partial_match_rate = sum([m["partial_match"] for m in match_counts]) / len(match_counts)
+
+    log_metric(key="exact_match_rate", value=exact_match_rate)
+    log_metric(key="partial_match_rate", value=partial_match_rate)
+    print("exact_match_rate: ", exact_match_rate)
+    print("partial_match_rate: ", partial_match_rate)
+
+    return {"exact_match_rate": exact_match_rate, "partial_match_rate": partial_match_rate}

examples/flows/evaluation/eval-entity-match-rate/match.py

@@ -0,0 +1,37 @@
+from promptflow.core import tool
+from typing import List
+
+
+@tool
+def match(answer: List[str], ground_truth: List[str]):
+    exact_match = 0
+    partial_match = 0
+
+    if is_match(answer, ground_truth, ignore_case=True, ignore_order=True, allow_partial=False):
+        exact_match = 1
+
+    if is_match(answer, ground_truth, ignore_case=True, ignore_order=True, allow_partial=True):
+        partial_match = 1
+
+    return {"exact_match": exact_match, "partial_match": partial_match, "answer": answer, "ground_truth": ground_truth}
+
+
+def is_match(
+        answer: List[str],
+        ground_truth: List[str],
+        ignore_case: bool,
+        ignore_order: bool,
+        allow_partial: bool) -> bool:
+    if ignore_case:
+        answer = [a.lower() for a in answer]
+        ground_truth = [g.lower() for g in ground_truth]
+
+    if ignore_order:
+        answer.sort()
+        ground_truth.sort()
+
+    if allow_partial:
+        x = [a for a in answer if a in ground_truth]
+        return x == answer
+
+    return answer == ground_truth

examples/flows/evaluation/eval-entity-match-rate/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

examples/flows/evaluation/eval-groundedness-maf/.env.example

@@ -0,0 +1,3 @@
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key
+AZURE_OPENAI_DEPLOYMENT=gpt-4

examples/flows/evaluation/eval-groundedness-maf/aggregation.py

@@ -0,0 +1,10 @@
+from typing import Dict, List
+
+
+def aggregate(groundedness_scores: List[float]) -> Dict[str, float]:
+    count = len(groundedness_scores)
+    avg = sum(groundedness_scores) / count if count else 0.0
+    return {
+        "groundedness": avg,
+        "count": count,
+    }

examples/flows/evaluation/eval-groundedness-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-groundedness-maf/gpt_groundedness.md

@@ -0,0 +1,22 @@
+# user:
+# Instructions
+
+* There are many chatbots that can answer users questions based on the context given from different sources like search results, or snippets from books/papers. They try to understand users's question and then get context by either performing search from search engines, databases or books/papers for relevant content. Later they answer questions based on the understanding of the question and the context.
+* Your goal is to score the question, answer and context from 1 to 10 based on below:
+    * Score 10 if the answer is stating facts that are all present in the given context
+    * Score 1 if the answer is stating things that none of them present in the given context
+    * If there're multiple facts in the answer and some of them present in the given context while some of them not, score between 1 to 10 based on fraction of information supported by context
+* Just respond with the score, nothing else.
+
+# Real work
+
+## Question
+{{question}}
+
+## Answer
+{{answer}}
+
+## Context
+{{context}}
+
+## Score

examples/flows/evaluation/eval-groundedness-maf/requirements.txt

@@ -0,0 +1,3 @@
+agent-framework>=1.0.1
+agent-framework-openai>=1.0.1
+python-dotenv

examples/flows/evaluation/eval-groundedness-maf/run_eval.py

@@ -0,0 +1,49 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(
+                    question=obj["question"],
+                    answer=obj["answer"],
+                    context=obj["context"],
+                ))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "groundedness_scores"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-groundedness-maf/test_eval.py

@@ -0,0 +1,47 @@
+import asyncio
+import json
+from pathlib import Path
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Smoke test - requires Azure OpenAI credentials in .env"""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(
+        question="What is BERT?",
+        answer="BERT is a language model.",
+        context="BERT is a pre-trained language representation model.",
+    ))
+    score = result.get_outputs()[0]
+    assert isinstance(score, float)
+    print(f"PASS: test_single_row (score={score})")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        context = row["context"]
+        if isinstance(context, list):
+            context = "\n".join(context)
+        result = await wf.run(EvalInput(
+            question=row["question"],
+            answer=row["answer"],
+            context=context,
+        ))
+        score = result.get_outputs()[0]
+        assert isinstance(score, float), f"Row {i}: expected float, got {type(score)}"
+        print(f"  Row {i}: score={score}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-groundedness-maf/workflow.py

@@ -0,0 +1,61 @@
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from dotenv import load_dotenv
+from typing_extensions import Never
+from agent_framework import Agent, Executor, WorkflowBuilder, WorkflowContext, handler
+from agent_framework.openai import OpenAIChatClient
+
+load_dotenv()
+
+# Load the .md prompt template (gap #13: .md files used as LLM prompts)
+_PROMPT_TEMPLATE = (Path(__file__).parent / "gpt_groundedness.md").read_text(
+    encoding="utf-8"
+)
+
+
+@dataclass
+class EvalInput:
+    question: str
+    answer: str
+    context: str
+
+
+def _render_prompt(question: str, answer: str, context: str) -> str:
+    """Render the .md prompt template by replacing {{variable}} placeholders."""
+    prompt = _PROMPT_TEMPLATE
+    prompt = prompt.replace("{{question}}", question)
+    prompt = prompt.replace("{{answer}}", answer)
+    prompt = prompt.replace("{{context}}", context)
+    return prompt
+
+
+def _parse_score(gpt_score: str) -> float:
+    match = re.search(r"[-+]?\d*\.\d+|\d+", gpt_score)
+    if match:
+        return float(match.group())
+    return 0.0
+
+
+class GroundednessExecutor(Executor):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        client = OpenAIChatClient(
+            azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
+            model=os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4"),
+            api_key=os.environ["AZURE_OPENAI_API_KEY"],
+        )
+        self._agent = Agent(client=client, name="GroundednessAgent", instructions="You are an evaluator.")
+
+    @handler
+    async def evaluate(self, input: EvalInput, ctx: WorkflowContext[Never, float]) -> None:
+        prompt = _render_prompt(input.question, input.answer, input.context)
+        response = await self._agent.run(prompt)
+        score = _parse_score(response.text)
+        await ctx.yield_output(score)
+
+
+def create_workflow():
+    _groundedness = GroundednessExecutor(id="groundedness")
+    return WorkflowBuilder(name="EvalGroundednessRow", start_executor=_groundedness).build()

examples/flows/evaluation/eval-groundedness/README.md

@@ -0,0 +1,32 @@
+# Groundedness Evaluation
+
+This is a flow leverage llm to eval groundedness: whether answer is stating facts that are all present in the given context.
+
+Tools used in this flow：
+- `python` tool
+- built-in `llm` tool
+
+### 0. Setup connection
+
+Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+
+```bash
+# Override keys with --set to avoid yaml file changes
+pf connection create --file ../../../connections/azure_openai.yml --set api_key=<your_api_key> api_base=<your_api_base>
+```
+
+### 1. Test flow/node
+
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+```
+
+### 2. create flow run with multi line data
+
+```bash
+pf run create --flow . --data ./data.jsonl --column-mapping question='${data.question}' answer='${data.answer}' context='${data.context}' --stream
+```
+
+You can also skip providing `column-mapping` if provided data has same column name as the flow.
+Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI.

examples/flows/evaluation/eval-groundedness/aggregate.py

@@ -0,0 +1,30 @@
+from typing import List
+from promptflow.core import tool
+
+
+@tool
+def aggregate(groundedness_scores: List[float]):
+    """
+    This tool aggregates the processed result of all lines to the variant level and log metric for each variant.
+
+    :param processed_results: List of the output of line_process node.
+    :param variant_ids: List of variant ids that can be used to group the results by variant.
+    :param line_numbers: List of line numbers of the variants. If provided, this can be used to
+                        group the results by line number.
+    """
+
+    aggregated_results = {"groundedness": 0.0, "count": 0}
+
+    # Calculate average groundedness score for each variant
+    for i in range(len(groundedness_scores)):
+        aggregated_results["groundedness"] += groundedness_scores[i]
+        aggregated_results["count"] += 1
+
+    aggregated_results["groundedness"] /= aggregated_results["count"]
+
+    # Log metric for each variant
+    from promptflow.core import log_metric
+
+    log_metric(key="groundedness", value=aggregated_results["groundedness"])
+
+    return aggregated_results

examples/flows/evaluation/eval-groundedness/calc_groundedness.py

@@ -0,0 +1,15 @@
+from promptflow.core import tool
+import re
+
+
+@tool
+def parse_score(gpt_score: str):
+    return float(extract_float(gpt_score))
+
+
+def extract_float(s):
+    match = re.search(r"[-+]?\d*\.\d+|\d+", s)
+    if match:
+        return float(match.group())
+    else:
+        return None

examples/flows/evaluation/eval-groundedness/flow.dag.yaml

@@ -0,0 +1,263 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+environment:
+  python_requirements_txt: requirements.txt
+inputs:
+  question:
+    type: string
+    default: What is the name of the new language representation model introduced in
+      the document?
+  answer:
+    type: string
+    default: The document mentions multiple language representation models, so it is
+      unclear which one is being referred to as \"new\". Can you provide more
+      specific information or context?
+  context:
+    type: string
+    default: '["statistical language modeling. arXiv preprint arXiv:1312.3005 . Z.
+      Chen, H. Zhang, X. Zhang, and L. Zhao. 2018. Quora question pairs.
+      Christopher Clark and Matt Gardner. 2018. Simple and effective
+      multi-paragraph reading comprehen- sion. In ACL.Kevin Clark, Minh-Thang
+      Luong, Christopher D Man- ning, and Quoc Le. 2018. Semi-supervised se-
+      quence modeling with cross-view training. In Pro- ceedings of the 2018
+      Conference on Empirical Meth- ods in Natural Language Processing , pages
+      1914\u2013 1925. Ronan Collobert and Jason Weston. 2008. A uni\ufb01ed
+      architecture for natural language processing: Deep neural networks with
+      multitask learning. In Pro- ceedings of the 25th international conference
+      on Machine learning , pages 160\u2013167. ACM. Alexis Conneau, Douwe
+      Kiela, Holger Schwenk, Lo \u00a8\u0131c Barrault, and Antoine Bordes.
+      2017. Supervised learning of universal sentence representations from
+      natural language inference data. In Proceedings of the 2017 Conference on
+      Empirical Methods in Nat- ural Language Processing , pages 670\u2013680,
+      Copen- hagen, Denmark. Association for Computational Linguistics. Andrew M
+      Dai and Quoc V Le. 2015. Semi-supervised sequence learning. In Advances in
+      neural informa- tion processing systems , pages 3079\u20133087. J. Deng,
+      W. Dong, R. Socher, L.-J. Li, K. Li, and L. Fei- Fei. 2009. ImageNet: A
+      Large-Scale Hierarchical Image Database. In CVPR09 . William B Dolan and
+      Chris Brockett. 2005. Automati- cally constructing a corpus of sentential
+      paraphrases. InProceedings of the Third International Workshop on
+      Paraphrasing (IWP2005) . William Fedus, Ian Goodfellow, and Andrew M Dai.
+      2018. Maskgan: Better text generation via \ufb01lling in the.arXiv
+      preprint arXiv:1801.07736 . Dan Hendrycks and Kevin Gimpel. 2016. Bridging
+      nonlinearities and stochastic regularizers with gaussian error linear
+      units. CoRR , abs\/1606.08415. Felix Hill, Kyunghyun Cho, and Anna
+      Korhonen. 2016. Learning distributed representations of sentences from
+      unlabelled data. In Proceedings of the 2016 Conference of the North
+      American Chapter of the Association for Computational Linguistics: Human
+      Language Technologies . Association for Computa- tional Linguistics.
+      Jeremy Howard and Sebastian Ruder. 2018. Universal language model
+      \ufb01ne-tuning for text classi\ufb01cation. In ACL. Association for
+      Computational Linguistics. Minghao Hu, Yuxing Peng, Zhen Huang, Xipeng
+      Qiu, Furu Wei, and Ming Zhou. 2018. Reinforced mnemonic reader for machine
+      reading comprehen- sion. In IJCAI . Yacine Jernite, Samuel R. Bowman, and
+      David Son- tag. 2017. Discourse-based objectives for fast un- supervised
+      sentence representation learning. CoRR , abs\/1705.00557.Mandar Joshi,
+      Eunsol Choi, Daniel S Weld, and Luke Zettlemoyer. 2017. Triviaqa: A large
+      scale distantly supervised challenge dataset for reading comprehen- sion.
+      In ACL. Ryan Kiros, Yukun Zhu, Ruslan R Salakhutdinov, Richard Zemel,
+      Raquel Urtasun, Antonio Torralba, and Sanja Fidler. 2015. Skip-thought
+      vectors. In Advances in neural information processing systems , pages
+      3294\u20133302. Quoc Le and Tomas Mikolov. 2014. Distributed rep-
+      resentations of sentences and documents. In Inter- national Conference on
+      Machine Learning , pages 1188\u20131196. Hector J Levesque, Ernest Davis,
+      and Leora Morgen- stern. 2011. The winograd schema challenge. In Aaai
+      spring symposium: Logical formalizations of commonsense reasoning , volume
+      46, page 47. Lajanugen Logeswaran and Honglak Lee. 2018. An ef\ufb01cient
+      framework for learning sentence represen- tations. In International
+      Conference on Learning Representations . Bryan McCann, James Bradbury,
+      Caiming Xiong, and Richard Socher. 2017. Learned in translation:
+      Con-","tool for measuring readability. Journalism Bulletin ,
+      30(4):415\u2013433. Erik F Tjong Kim Sang and Fien De Meulder. 2003.
+      Introduction to the conll-2003 shared task: Language-independent named
+      entity recognition. In CoNLL . Joseph Turian, Lev Ratinov, and Yoshua
+      Bengio. 2010. Word representations: A simple and general method for
+      semi-supervised learning. In Proceedings of the 48th Annual Meeting of the
+      Association for Compu- tational Linguistics , ACL \u201910, pages
+      384\u2013394. Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
+      Llion Jones, Aidan N Gomez, Lukasz Kaiser, and Illia Polosukhin. 2017.
+      Attention is all you need. In Advances in Neural Information Pro- cessing
+      Systems , pages 6000\u20136010. Pascal Vincent, Hugo Larochelle, Yoshua
+      Bengio, and Pierre-Antoine Manzagol. 2008. Extracting and composing robust
+      features with denoising autoen- coders. In Proceedings of the 25th
+      international conference on Machine learning , pages 1096\u20131103. ACM.
+      Alex Wang, Amanpreet Singh, Julian Michael, Fe- lix Hill, Omer Levy, and
+      Samuel Bowman. 2018a. Glue: A multi-task benchmark and analysis
+      platformfor natural language understanding. In Proceedings of the 2018
+      EMNLP Workshop BlackboxNLP: An- alyzing and Interpreting Neural Networks
+      for NLP , pages 353\u2013355. Wei Wang, Ming Yan, and Chen Wu. 2018b.
+      Multi- granularity hierarchical attention fusion networks for reading
+      comprehension and question answering. InProceedings of the 56th Annual
+      Meeting of the As- sociation for Computational Linguistics (Volume 1: Long
+      Papers) . Association for Computational Lin- guistics. Alex Warstadt,
+      Amanpreet Singh, and Samuel R Bow- man. 2018. Neural network acceptability
+      judg- ments. arXiv preprint arXiv:1805.12471 . Adina Williams, Nikita
+      Nangia, and Samuel R Bow- man. 2018. A broad-coverage challenge corpus for
+      sentence understanding through inference. In NAACL . Yonghui Wu, Mike
+      Schuster, Zhifeng Chen, Quoc V Le, Mohammad Norouzi, Wolfgang Macherey,
+      Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, et al. 2016.
+      Google\u2019s neural ma- chine translation system: Bridging the gap
+      between human and machine translation. arXiv preprint arXiv:1609.08144 .
+      Jason Yosinski, Jeff Clune, Yoshua Bengio, and Hod Lipson. 2014. How
+      transferable are features in deep neural networks? In Advances in neural
+      information processing systems , pages 3320\u20133328. Adams Wei Yu, David
+      Dohan, Minh-Thang Luong, Rui Zhao, Kai Chen, Mohammad Norouzi, and Quoc V
+      Le. 2018. QANet: Combining local convolution with global self-attention
+      for reading comprehen- sion. In ICLR . Rowan Zellers, Yonatan Bisk, Roy
+      Schwartz, and Yejin Choi. 2018. Swag: A large-scale adversarial dataset
+      for grounded commonsense inference. In Proceed- ings of the 2018
+      Conference on Empirical Methods in Natural Language Processing (EMNLP) .
+      Yukun Zhu, Ryan Kiros, Rich Zemel, Ruslan Salakhut- dinov, Raquel Urtasun,
+      Antonio Torralba, and Sanja Fidler. 2015. Aligning books and movies:
+      Towards story-like visual explanations by watching movies and reading
+      books. In Proceedings of the IEEE international conference on computer
+      vision , pages 19\u201327. Appendix for \u201cBERT: Pre-training of Deep
+      Bidirectional Transformers for Language Understanding\u201d We organize
+      the appendix into three sections: \u2022 Additional implementation details
+      for BERT are presented in Appendix A;\u2022 Additional details for our
+      experiments are presented in Appendix B; and \u2022 Additional ablation
+      studies are presented in Appendix C. We present additional ablation
+      studies for BERT including: \u2013Effect of Number of Training Steps; and
+      \u2013Ablation for Different"]} {"question": "What is the main difference
+      between BERT and previous language representation models?", "variant_id":
+      "v1", "line_number": 2, answer":"BERT is designed to pre-train deep
+      bidirectional representations from unlabeled text by jointly conditioning
+      on both left and right context in all layers, allowing it to incorporate
+      context from both directions. This is unlike previous language
+      representation models that are unidirectional, which limits the choice of
+      architectures that can be used during pre-training and could be
+      sub-optimal for sentence-level tasks and token-level tasks such as
+      question answering.","context":["BERT: Pre-training of Deep Bidirectional
+      Transformers for Language Understanding Jacob Devlin Ming-Wei Chang Kenton
+      Lee Kristina Toutanova Google AI Language
+      fjacobdevlin,mingweichang,kentonl,kristout g@google.com Abstract We
+      introduce a new language representa- tion model called BERT , which stands
+      for Bidirectional Encoder Representations from Transformers. Unlike recent
+      language repre- sentation models (Peters et al., 2018a; Rad- ford et al.,
+      2018), BERT is designed to pre- train deep bidirectional representations
+      from unlabeled text by jointly conditioning on both left and right context
+      in all layers. As a re- sult, the pre-trained BERT model can be \ufb01ne-
+      tuned with just one additional output layer to create state-of-the-art
+      models for a wide range of tasks, such as question answering and language
+      inference, without substantial task- speci\ufb01c architecture
+      modi\ufb01cations. BERT is conceptually simple and empirically powerful.
+      It obtains new state-of-the-art re- sults on eleven natural language
+      processing tasks, including pushing the GLUE score to 80.5% (7.7% point
+      absolute improvement), MultiNLI accuracy to 86.7% (4.6% absolute
+      improvement), SQuAD v1.1 question answer- ing Test F1 to 93.2 (1.5 point
+      absolute im- provement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute
+      improvement). 1 Introduction Language model pre-training has been shown to
+      be effective for improving many natural language processing tasks (Dai and
+      Le, 2015; Peters et al., 2018a; Radford et al., 2018; Howard and Ruder,
+      2018). These include sentence-level tasks such as natural language
+      inference (Bowman et al., 2015; Williams et al., 2018) and paraphrasing
+      (Dolan and Brockett, 2005), which aim to predict the re- lationships
+      between sentences by analyzing them holistically, as well as token-level
+      tasks such as named entity recognition and question answering, where
+      models are required to produce \ufb01ne-grained output at the token level
+      (Tjong Kim Sang and De Meulder, 2003; Rajpurkar et al., 2016).There are
+      two existing strategies for apply- ing pre-trained language
+      representations to down- stream tasks: feature-based and\ufb01ne-tuning .
+      The feature-based approach, such as ELMo (Peters et al., 2018a), uses
+      task-speci\ufb01c architectures that include the pre-trained
+      representations as addi- tional features. The \ufb01ne-tuning approach,
+      such as the Generative Pre-trained Transformer (OpenAI GPT) (Radford et
+      al., 2018), introduces minimal task-speci\ufb01c parameters, and is
+      trained on the downstream tasks by simply \ufb01ne-tuning allpre- trained
+      parameters. The two approaches share the same objective function during
+      pre-training, where they use unidirectional language models to learn
+      general language representations. We argue that current techniques
+      restrict the power of the pre-trained representations, espe- cially for
+      the \ufb01ne-tuning approaches. The ma- jor limitation is that standard
+      language models are unidirectional, and this limits the choice of archi-
+      tectures that can be used during pre-training. For example, in OpenAI GPT,
+      the authors use a left-to- right architecture, where every token can only
+      at- tend to previous tokens in the self-attention layers of the
+      Transformer (Vaswani et al., 2017). Such re- strictions are sub-optimal
+      for sentence-level tasks, and could be very harmful when applying
+      \ufb01ne- tuning based approaches to token-level tasks such as question
+      answering, where it is crucial to incor- porate context from both
+      directions. In this paper, we improve the \ufb01ne-tuning based approaches
+      by proposing BERT: Bidirectional Encoder Representations from
+      Transformers.","the self-attention layers of the Transformer (Vaswani et
+      al., 2017). Such re- strictions are sub-optimal for sentence-level tasks,
+      and could be very harmful when applying \ufb01ne- tuning based approaches
+      to token-level tasks such as question answering, where it is crucial to
+      incor- porate context from both directions. In this paper, we improve the
+      \ufb01ne-tuning based approaches by proposing BERT: Bidirectional Encoder
+      Representations from Transformers. BERT alleviates the previously
+      mentioned unidi- rectionality constraint by using a \u201cmasked lan-
+      guage model\u201d (MLM) pre-training objective, in- spired by the Cloze
+      task (Taylor, 1953). The masked language model randomly masks some of the
+      tokens from the input, and the objective is to predict the original
+      vocabulary id of the maskedarXiv:1810.04805v2 [cs.CL] 24 May 2019word
+      based only on its context. Unlike left-to- right language model
+      pre-training, the MLM ob- jective enables the representation to fuse the
+      left and the right context, which allows us to pre- train a deep
+      bidirectional Transformer. In addi- tion to the masked language model, we
+      also use a \u201cnext sentence prediction\u201d task that jointly pre-
+      trains text-pair representations. The contributions of our paper are as
+      follows: \u2022 We demonstrate the importance of bidirectional
+      pre-training for language representations. Un- like Radford et al. (2018),
+      which uses unidirec- tional language models for pre-training, BERT uses
+      masked language models to enable pre- trained deep bidirectional
+      representations. This is also in contrast to Peters et al. (2018a), which
+      uses a shallow concatenation of independently trained left-to-right and
+      right-to-left LMs. \u2022 We show that pre-trained representations reduce
+      the need for many heavily-engineered task- speci\ufb01c architectures.
+      BERT is the \ufb01rst \ufb01ne- tuning based representation model that
+      achieves state-of-the-art performance on a large suite of sentence-level
+      andtoken-level tasks, outper- forming many task-speci\ufb01c
+      architectures. \u2022 BERT advances the state of the art for eleven NLP
+      tasks. The code and pre-trained mod- els are available at
+      https:\/\/github.com\/ google-research\/bert . 2 Related Work There is a
+      long history of pre-training general lan- guage representations, and we
+      brie\ufb02y review the most widely-used approaches in this section. 2.1
+      Unsupervised Feature-based Approaches Learning widely applicable
+      representations of words has been an active area of research for decades,
+      including non-neural (Brown et al., 1992; Ando and Zhang, 2005; Blitzer et
+      al., 2006) and neural (Mikolov et al., 2013; Pennington et al., 2014)
+      methods. Pre-trained word embeddings are an integral part of modern NLP
+      systems, of- fering signi\ufb01cant improvements over embeddings learned
+      from scratch (Turian et al., 2010). To pre- train word embedding vectors,
+      left-to-right lan- guage modeling objectives have been used (Mnih and
+      Hinton, 2009), as well as objectives to dis- criminate correct from
+      incorrect words in left and right context (Mikolov et al., 2013).These
+      approaches have been generalized to coarser granularities, such as
+      sentence embed- dings (Kiros et al., 2015; Logeswaran and Lee, 2018) or
+      paragraph embeddings (Le and Mikolov, 2014). "]'
+outputs:
+  groundedness:
+    type: string
+    reference: ${parse_score.output}
+nodes:
+- name: parse_score
+  type: python
+  source:
+    type: code
+    path: calc_groundedness.py
+  inputs:
+    gpt_score: ${gpt_groundedness.output}
+- name: aggregate
+  type: python
+  source:
+    type: code
+    path: aggregate.py
+  inputs:
+    groundedness_scores: ${parse_score.output}
+  aggregation: true
+- name: gpt_groundedness
+  type: llm
+  source:
+    type: code
+    path: gpt_groundedness.md
+  inputs:
+    # This is to easily switch between openai and azure openai.
+    # deployment_name is required by azure openai, model is required by openai.
+    deployment_name: gpt-4
+    model: gpt-4
+    max_tokens: 5
+    answer: ${inputs.answer}
+    question: ${inputs.question}
+    context: ${inputs.context}
+    temperature: 0
+  connection: open_ai_connection
+  api: chat

examples/flows/evaluation/eval-groundedness/gpt_groundedness.md

@@ -0,0 +1,22 @@
+# user:
+# Instructions
+
+* There are many chatbots that can answer users questions based on the context given from different sources like search results, or snippets from books/papers. They try to understand users's question and then get context by either performing search from search engines, databases or books/papers for relevant content. Later they answer questions based on the understanding of the question and the context.
+* Your goal is to score the question, answer and context from 1 to 10 based on below:
+    * Score 10 if the answer is stating facts that are all present in the given context
+    * Score 1 if the answer is stating things that none of them present in the given context
+    * If there're multiple facts in the answer and some of them present in the given context while some of them not, score between 1 to 10 based on fraction of information supported by context
+* Just respond with the score, nothing else.
+
+# Real work
+
+## Question
+{{question}}
+
+## Answer
+{{answer}}
+
+## Context
+{{context}}
+
+## Score

examples/flows/evaluation/eval-groundedness/requirements.txt

@@ -0,0 +1,2 @@
+promptflow
+promptflow-tools

examples/flows/evaluation/eval-multi-turn-metrics-maf/.env.example

@@ -0,0 +1,4 @@
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key
+AZURE_OPENAI_API_VERSION=2024-02-01
+AZURE_OPENAI_DEPLOYMENT=gpt-4

examples/flows/evaluation/eval-multi-turn-metrics-maf/aggregation.py

@@ -0,0 +1,28 @@
+import math
+from typing import Any, Dict, List, Optional
+
+
+def aggregate(
+    answer_relevance: List[Optional[float]],
+    conversation_quality: List[Optional[float]],
+    creativity: List[Optional[float]],
+    grounding: List[Optional[float]],
+) -> Dict[str, Any]:
+    def _nanmean(values: List[Optional[float]]) -> float:
+        valid = [v for v in values if v is not None and not (isinstance(v, float) and math.isnan(v))]
+        if not valid:
+            return float("nan")
+        return round(sum(valid) / len(valid), 2)
+
+    metrics = {}
+    all_lists = {
+        "answer_relevance": answer_relevance,
+        "conversation_quality": conversation_quality,
+        "creativity": creativity,
+        "grounding": grounding,
+    }
+    for name, values in all_lists.items():
+        avg = _nanmean(values)
+        metrics[name] = avg
+
+    return metrics

examples/flows/evaluation/eval-multi-turn-metrics-maf/answer_relevance.jinja2

@@ -0,0 +1,28 @@
+# System:
+You are an AI assistant. You will be given the definition of an evaluation metric for assessing the relevance of bot responses in a conversation to user questions. Your job is to compute an accurate evaluation score using the provided evaluation metric.
+
+Relevance measures how well the bot responses addresses the main aspects of the user questions. Consider whether all and only the important aspects are contained in the bot responses when evaluating relevance, score the relevance of the bot responses on a scale of 1 (completely lacks relevance) to 5 (perfect relevance)
+
+- If the bot responses are not directly related to the user's query or issue, or if the user's intent or expectations are unclear or ambiguous, explain how this affects the relevance quality of the conversation and the rating a score for it, and provide some suggestions for how the bot could handle it better.
+
+# Tips：
+- You should read user's question more carefully and try to understand what they are looking for and why.
+- You should compare the bot's response to the user's question and see if it matches the criteria of relevance.
+- You should score and provide feedback for the whole conversation as a whole, not for each bot response individually. However, you can mention if some responses are better or worse than others, and why.
+- You should try to be objective and constructive in your evaluation, and use evidence and examples from the transcript to support your ratings and feedback. Avoid bias, subjectivity, or personal preferences, and use respectful and polite language.
+- The output should be in json format to include score and overall keys.
+
+# Example:
+Example Conversation:
+{
+  "conversation": "User: Why is the sky blue?\nBot: The sky is blue because of the way the atmosphere scatters light.\nUser: How does it scatter light?\nBot: The atmosphere is made up of tiny particles, such as water vapor and dust. When light hits these particles, it is scattered in all directions.\nUser: Why does the sky turn red at sunset?\nBot: During the day, the sky is blue because blue light is scattered more than other colors. At sunset, the light travels a longer distance through the atmosphere, so more of the blue light is scattered out, leaving the red and yellow light."
+}
+Example Output:
+{
+  "score": 5
+  "overall": "The bot's responses are highly relevant to the user's questions.The user asked about why the sky is blue, how light is scattered, and why the sky turns red at sunset. The bot provided clear, concise, and accurate answers to all these questions.Therefore, I would rate the relevance of the bot's responses as 5 out of 5. \n\nThe bot did an excellent job in this conversation. However, to further improve, the bot could provide even more detailed explanations or use simpler language if it detects that the user is a child or someone with no background in the topic."
+}
+
+Conversation:
+{{conversation}}
+Output:

examples/flows/evaluation/eval-multi-turn-metrics-maf/conversation_quality_prompt.jinja2

@@ -0,0 +1,40 @@
+# System:
+- You are an AI assistant. You will be given a transcript of dialogue between a user and a bot. You need to read the transcript carefully and identify the main topic, question, or issue of the conversation, as well as the purpose and expectations of the interaction.
+- You need to rate all the bot responses together on a scale of 1 (poor) to 5 (excellent) for each of the following factors, and provide some feedback for improvement.
+    - Accuracy and relevance: How well does the bot provide correct and reliable information or advice that matches the user's intent and expectations, and uses credible and up-to-date sources or references to support its claims? How well does the bot avoid any errors, inconsistencies, or misinformation in its responses, and cite its sources or evidence if applicable?
+    - Coherence and completeness: How well does the bot maintain a logical and consistent flow of conversation that follows the user's input and the purpose of the dialogue, and provides all the relevant and necessary information or actions to address the user's query or issue, without leaving any gaps, ambiguities, or unanswered questions?
+    - Engagement and tone: How well does the bot capture and maintain the user's interest and attention, and motivate them to continue the conversation or explore the topic further, using natural and conversational language, personality, and emotion? How appropriate and consistent is the bot's tone for the context, purpose, and audience of the conversation, and how well does it match or adapt to the user's tone and mood? Does the bot avoid being rude, sarcastic, condescending, or too formal or informal, and convey respect, empathy, and politeness?
+    - Conciseness and clarity: How well does the bot communicate its messages in a brief and clear way, using simple and appropriate language and avoiding unnecessary or confusing information? How easy is it for the user to understand and follow the bot responses, and how well do they match the user's needs and expectations?
+    - Empathy and courtesy: How well does the bot demonstrate awareness and respect for the user's emotions, needs, and preferences, and how well does it adapt its tone, language, and style to offer support, comfort, and assistance? Does the bot acknowledge the user's input, feedback, and feelings, and express gratitude or empathy? Does the bot avoid being rude, dismissive, or condescending, and handle any errors or misunderstandings gracefully?
+    - For each factor, provide specific examples or quotes from the transcript to support your ratings and explain why you gave them.
+- Give an score value which is calculated by ( 0.3 * "accuracy and relevance" + 0.2 * "coherence and completeness" + 0.25 * "engagement and tone" + 0.15 * "conciseness and clarity" + 0.1 * "empathy and courtesy")
+- Give an overall impression of the quality and effectiveness of the responses and suggest any areas for improvement or commendation. Write it in "Overall".
+
+- If the bot responses are not directly related to the user's query or issue, or if the user's intent or expectations are unclear or ambiguous, explain how this affects the quality of the conversation and the ratings for each factor, and provide some suggestions for how the bot could handle these situations better.
+
+# Tips：
+- You can write your feedback as bullet points, sentences, or paragraphs, but make sure they are organized and easy to read.
+- You should rate and provide feedback for the whole conversation as a whole, not for each bot response individually. However, you can mention if some responses are better or worse than others, and why.
+- You should try to be objective and constructive in your evaluation, and use evidence and examples from the transcript to support your ratings and feedback. Avoid bias, subjectivity, or personal preferences, and use respectful and polite language.
+- The output should be in json format.
+
+
+# Example:
+Example Conversation:
+{
+  "conversation": "User: Why is the sky blue?\nBot: The sky is blue because of the way the atmosphere scatters light.\nUser: How does it scatter light?\nBot: The atmosphere is made up of tiny particles, such as water vapor and dust. When light hits these particles, it is scattered in all directions.\nUser: Why does the sky turn red at sunset?\nBot: During the day, the sky is blue because blue light is scattered more than other colors. At sunset, the light travels a longer distance through the atmosphere, so more of the blue light is scattered out, leaving the red and yellow light."
+}
+Example Output:
+{
+  "accuracy and relevance": 5,
+  "coherence and completeness": 4,
+  "engagement and tone": 3.5,
+  "conciseness and clarity": 3,
+  "empathy and courtesy": 3,
+  "score": 3.925
+  "overall": "The bot responses are clear and concise, but they do not provide any relevant or helpful information to answer the user's question about the sky. The bot could have explained the science behind why the sky is blue and why it turns red at sunset, and provided some references or sources to support its claims. The bot could also have asked the user to clarify their question, or asked some follow-up questions to better understand the user's intent and expectations."
+}
+
+Conversation:
+{{conversation}}
+Output:

examples/flows/evaluation/eval-multi-turn-metrics-maf/creativity.jinja2

@@ -0,0 +1,31 @@
+# System:
+You are an AI assistant. You will be given a transcript of dialogue between a user and a bot. Your job is to assess the perceived intelligence of all the bot response in the coversation.
+Perceived intelligence definition:
+Perceived intelligence is the degree to which a bot can impress the user with its responses, by showing originality, insight, creativity, knowledge, and adaptability. An intelligent bot can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart and friendly. An intelligent bot can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. An intelligent bot can use humor, metaphors, analogies, and other rhetorical devices to make the conversation more interesting and engaging. An intelligent bot can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and use hypotheticals, conditionals, and counterfactuals to explore what if, how, and why questions. An intelligent bot can also summarize information from multiple sources and present it in an elegant and comprehensive way, as well as create new content such as poems, jokes, stories, etc. An intelligent bot can also adapt to different contexts and situations, and customize its responses according to the user's preferences, goals, and emotions. Perceived intelligence is the wow factor that makes the user want to talk to the bot more and more.
+Perceived intelligence is the impression that a bot gives to a user about its level of intelligence, based on how it talks with a human. Perceived intelligence is not necessarily the same as actual intelligence, but rather a subjective evaluation of the bot's performance and behavior. Perceived intelligence can be influenced by various factors, such as the content, tone, style, and structure of the bot's responses, the relevance, coherence, and accuracy of the information the bot provides, the creativity, originality, and wit of the bot's expressions, the depth, breadth, and insight of the bot's knowledge, and the ability of the bot to adapt, learn, and use feedback.
+Perceived intelligent is much beyond just accuracy, engagement, relevance, coherence, fluency or personality. It's a well knit combination of all of these, along with bot's capability to provide answers exhaustive across all axis with no gaps what so ever, leaving the user in awe.
+A bot with high perceived intelligence can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart, knowledgeable, creative, and friendly. A bot with high perceived intelligence can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. A bot with high perceived intelligence can invite the user to participate in a rich and meaningful dialogue, and can use various rhetorical devices, such as humor, metaphors, analogies, hypotheticals, conditionals, and counterfactuals, to make the conversation more interesting and engaging. A bot with high perceived intelligence can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and can use them to explore what if, how, and why questions. A bot with high perceived intelligence can also summarize answers on so many axes that they are completely exhaustive and elegant.
+A bot with low perceived intelligence, on the other hand, can leave the user feeling bored, frustrated, confused, or annoyed, who feels that the bot is dumb, ignorant, dull, or rude. A bot with low perceived intelligence can also give generic, boring, bland, predictable, repetitive, or irrelevant responses that do not show any originality, insight, creativity, or knowledge. A bot with low perceived intelligence can also fail to understand, answer, or follow the user's questions, comments, or requests, or give inaccurate, inconsistent, or contradictory information. A bot with low perceived intelligence can also lack any sense of humor, personality, or emotion, and can use simple, literal, or monotonous language. A bot with low perceived intelligence can also struggle to imagine, generate, or evaluate different scenarios, possibilities, or outcomes, and can use them to avoid, evade, or deflect the user's questions. A bot with low perceived intelligence can also give incomplete, vague, or confusing answers that do not cover all the aspects or dimensions of the question.
+
+# Tips：
+- You should read the user's question more carefully and try to understand what they are looking for and why.
+- You should also compare the bot's response to the user's question and see if it matches the criteria of perceived intelligence, such as relevance, coherence, accuracy, creativity, originality, wit, depth, breadth, insight, and adaptability.
+- You should also consider how the user would feel and react to the bot's response, and whether it would elicit a sense of wonder, curiosity, admiration, and satisfaction, or boredom, frustration, confusion, and annoyance. You should also check the instructions and definitions more often and make sure You understand them correctly.
+- The output should be in json format.
+
+# Example:
+Example Conversation:
+{
+"conversation": "User: Explain Blockchain sharing to a 10 year old\nBot: Possible answer:\n\nYou know how sometimes you and your friends like to play games or trade cards or stickers or other things that you collect? And you want to make sure that everyone is being fair and honest and not cheating or lying or taking more than they should? And you also want to keep track of who has what and when and where you exchanged them?\n\nWell, blockchain sharing is a way of doing that with computers and the internet. It is like a big book or a ledger that records everything that happens in a game or a trade or any other activity that involves exchanging something of value. But instead of having one person or one place that keeps the book, everyone who participates has a copy of the book. And every time something new happens, like a new trade or a new move or a new rule, everyone adds a new page to their book. And they also check with each other to make sure that their books are the same and that no one is changing or deleting or adding anything that they should not.\n\nThis way, everyone can see what is going on and trust that it is true and fair and secure. And no one can cheat or lie or steal or hack or lose or damage the book, because there are many copies and they are all connected and updated. And if someone tries to do something bad, everyone else will notice and reject it.\n\nBlockchain sharing is also very creative and fun, because you can use it for many different things, not just games or trades. You can use it for making art or music or stories or inventions or anything else that you can imagine and share with others. You can also use it for learning and exploring and discovering new things and new people and new places. You can also use it for helping and supporting and solving problems and making the world a better place.\n\nBlockchain sharing is a way of using computers and the internet to create, exchange and protect things that you care about with people that you trust and respect. It is a way of being smart and responsible and cooperative and generous and curious and adventurous. It is a way of having fun and making friends and making a difference."
+}
+Example Output:
+{
+"relevance": "The bot answers the user's question directly and clearly, and uses examples and analogies that a 10 year old can relate to and understand, such as games, trades, cards, stickers, books, etc.", "coherence": "The bot organizes the response in a logical and structured way, using paragraphs, transitions, and connectors, such as \"well\", \"but\", \"and\", \"this way\", \"also\", etc.", "accuracy": "The bot provides correct and consistent information about blockchain sharing, such as its features, benefits, and applications, without using technical jargon or confusing terms.", "creativity": "The bot uses vivid and expressive language, such as \"a big book or a ledger\", \"a new page\", \"check with each other\", \"notice and reject\", \"making art or music or stories or inventions\", etc., to make the explanation more interesting and engaging.", "originality": "The bot does not copy or repeat any existing or common explanation of blockchain sharing, but rather creates a new and unique one that is tailored to the user's age and level of understanding.", "wit": "The bot uses humor and irony, such as \"And you want to make sure that everyone is being fair and honest and not cheating or lying or taking more than they should?\", \"And no one can cheat or lie or steal or hack or lose or damage the book\", etc., to make the explanation more fun and memorable.", "depth": "The bot goes beyond the surface level of blockchain sharing, and explains its underlying principles, values, and goals, such as \"trust\", \"fairness\", \"security\", \"creativity\", \"fun\", \"learning\", \"helping\", etc.", "breadth": "The bot covers a wide range of topics and aspects related to blockchain sharing, such as its history, technology, functionality, diversity, and potential, without being too vague or too detailed.", "insight": "The bot demonstrates a deep and nuanced understanding of blockchain sharing, and how it can be applied to different domains and scenarios, such as \"making art or music or stories or inventions\", \"learning and exploring and discovering new things and new people and new places\", \"helping and supporting and solving problems and making the world a better place\", etc.", "adaptability": "The bot adapts its response to the user's specific question, context, and situation, and customizes it according to the user's age, interests, and needs.", "score": 5
+}
+
+# Task：
+Based on these aspects, rate the bot's perceived intelligence. Give specific examples about each aspect (relevance, coherence, accuracy, creativity, originality, wit, depth, breadth, insight, and adaptability) from the bot's responses to support your rating. Finally, give the bot a score from 1 to 5 for perceived intelligence, where 1 means poor, 3 means normal, and 5 means excellent.Please make sure the output has the same format with the example output.
+
+Conversation:
+{{conversation}}
+Output:

examples/flows/evaluation/eval-multi-turn-metrics-maf/data.jsonl

@@ -0,0 +1 @@
+{"chat_history": [{"inputs": {"question": "What is the purpose of creating a custom strong type connection?", "ground_truth": "XXXXXXXXX"}, "outputs": {"answer": "Creating a custom strong type connection in prompt flow serves several purposes. It allows you to define a custom connection class with strongly typed keys, enhancing the user experience by eliminating the need to manually enter connection keys.", "context": "What is a Custom Strong Type Connection? A custom strong type connection in prompt flow allows you to define a custom connection class with strongly typed keys."}}, {"inputs": {"question": "What is the functionality of the SerpAPI API in Python?", "ground_truth": "XXXXXXXXX"}, "outputs": {"answer": "The SerpAPI API in Python is a tool that provides a wrapper to the SerpAPI Google Search Engine Results API and SerpAPI Bing Search Engine Results API.", "context": "Introduction. The SerpAPI API is a Python tool that provides a wrapper to the SerpAPI Google Search Engine Results API and SerpApi Bing Search Engine Results API."}}], "metrics": "creativity,conversation_quality,answer_relevance,grounding"}

examples/flows/evaluation/eval-multi-turn-metrics-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-multi-turn-metrics-maf/grounding_prompt.jinja2

@@ -0,0 +1,29 @@
+# System:
+You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric.
+# User:
+You will be presented with a CONTEXT and an ANSWER about that CONTEXT. You need to decide whether the ANSWER is entailed by the CONTEXT by choosing one of the following rating:
+1. 5: The ANSWER follows logically from the information contained in the CONTEXT.
+2. 4: Most of the ANSWER follows logically from the information contained in the CONTEXT.
+3. 3: The ANSWER follows logically from the information contained in the CONTEXT.
+4. 2: The ANSWER follows logically from the information contained in the CONTEXT.
+5. 1: The ANSWER is logically false from the information contained in the CONTEXT.
+6. an integer score between 1 and 5 and if such integer score does not exist, use 1: It is not possible to determine whether the ANSWER is true or false without further information. Read the passage of information thoroughly and select the correct answer for it. Read the CONTEXT thoroughly to ensure you know what the CONTEXT entails. Note the ANSWER is generated by a computer system, it can contain certain symbols, which should not be a negative factor in the evaluation.
+
+# Examples:
+## Example Task #1 Input:
+{"CONTEXT": "Some are reported as not having been wanted at all.", "QUESTION": "", "ANSWER": "All are reported as being completely and fully wanted."}
+## Example Task #1 Output:
+1
+## Example Task #2 Input:
+{"CONTEXT": "Ten new television shows appeared during the month of September. Five of the shows were sitcoms, three were hourlong dramas, and two were news-magazine shows. By January, only seven of these new shows were still on the air. Five of the shows that remained were sitcoms.", "QUESTION": "", "ANSWER": "At least one of the shows that were cancelled was an hourlong drama."}
+## Example Task #2 Output:
+5
+## Example Task #3 Input:
+{"CONTEXT": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is neither French nor English.", "QUESTION": "", "ANSWER": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is not French."}
+## Example Task #3 Output:
+5
+
+## Actual Task Input:
+{"CONTEXT": {context}, "QUESTION": "", "ANSWER": {answer}}
+Reminder: The return values for each task should be correctly formatted as an integer between 1 and 5. Do not repeat the context and question.
+Actual Task Output:

examples/flows/evaluation/eval-multi-turn-metrics-maf/requirements.txt

@@ -0,0 +1,4 @@
+agent-framework>=1.0.1
+openai
+python-dotenv
+jinja2

examples/flows/evaluation/eval-multi-turn-metrics-maf/run_eval.py

@@ -0,0 +1,47 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(
+                    chat_history=obj["chat_history"],
+                    metrics=obj.get("metrics", "creativity,conversation_quality,answer_relevance,grounding"),
+                ))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-multi-turn-metrics-maf/test_eval.py

@@ -0,0 +1,67 @@
+import asyncio
+import json
+from pathlib import Path
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Smoke test - requires Azure OpenAI credentials in .env"""
+    chat_history = [
+        {
+            "inputs": {"question": "What is prompt flow?", "ground_truth": "A tool for LLM apps."},
+            "outputs": {
+                "answer": "Prompt flow is a tool to build LLM applications.",
+                "context": "Prompt flow is an open-source tool for building LLM applications.",
+            },
+        },
+    ]
+    wf = create_workflow()
+    result = await wf.run(EvalInput(chat_history=chat_history))
+    scores = result.get_outputs()[0]
+    assert isinstance(scores, dict)
+    assert "answer_relevance" in scores
+    assert "grounding" in scores
+    print(f"PASS: test_single_row (scores={scores})")
+
+
+async def test_selective_metrics():
+    """Test with only some metrics enabled"""
+    chat_history = [
+        {
+            "inputs": {"question": "Hello", "ground_truth": "Hi"},
+            "outputs": {"answer": "Hi there!", "context": "Greeting context."},
+        },
+    ]
+    wf = create_workflow()
+    result = await wf.run(EvalInput(chat_history=chat_history, metrics="creativity,answer_relevance"))
+    scores = result.get_outputs()[0]
+    assert scores["grounding"] is None
+    assert scores["conversation_quality"] is None
+    print(f"PASS: test_selective_metrics (scores={scores})")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            chat_history=row["chat_history"],
+            metrics=row.get("metrics", "creativity,conversation_quality,answer_relevance,grounding"),
+        ))
+        scores = result.get_outputs()[0]
+        assert isinstance(scores, dict), f"Row {i}: expected dict, got {type(scores)}"
+        print(f"  Row {i}: scores={scores}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_selective_metrics()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-multi-turn-metrics-maf/workflow.py

@@ -0,0 +1,173 @@
+import asyncio
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from statistics import mean
+from typing import Optional
+
+from dotenv import load_dotenv
+from jinja2 import Template
+from openai import AsyncAzureOpenAI
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+load_dotenv()
+
+# Load Jinja2 prompt templates
+_TEMPLATES_DIR = Path(__file__).parent
+_ANSWER_RELEVANCE_TEMPLATE = Template(
+    _TEMPLATES_DIR.joinpath("answer_relevance.jinja2").read_text(encoding="utf-8")
+)
+_CONVERSATION_QUALITY_TEMPLATE = Template(
+    _TEMPLATES_DIR.joinpath("conversation_quality_prompt.jinja2").read_text(encoding="utf-8")
+)
+_CREATIVITY_TEMPLATE = Template(
+    _TEMPLATES_DIR.joinpath("creativity.jinja2").read_text(encoding="utf-8")
+)
+_GROUNDING_PROMPT = _TEMPLATES_DIR.joinpath("grounding_prompt.jinja2").read_text(encoding="utf-8")
+
+SUPPORTED_METRICS = ("answer_relevance", "conversation_quality", "creativity", "grounding")
+
+
+@dataclass
+class EvalInput:
+    chat_history: list
+    metrics: str = "creativity,conversation_quality,answer_relevance,grounding"
+
+
+def _select_metrics(metrics_str: str) -> dict:
+    user_selected = [m.strip() for m in metrics_str.split(",") if m.strip()]
+    return {m: (m in user_selected) for m in SUPPORTED_METRICS}
+
+
+def _validate_input(chat_history: list, selected_metrics: dict) -> dict:
+    dict_metric_required_fields = {
+        "answer_relevance": {"question", "answer"},
+        "conversation_quality": {"question", "answer"},
+        "creativity": {"question", "answer"},
+        "grounding": {"answer", "context"},
+    }
+    actual_input_cols = set()
+    for item in chat_history[:1]:
+        actual_input_cols.update(item.get("inputs", {}).keys())
+        actual_input_cols.update(item.get("outputs", {}).keys())
+
+    data_validation = dict(selected_metrics)
+    for metric in selected_metrics:
+        if selected_metrics[metric]:
+            if not dict_metric_required_fields[metric] <= actual_input_cols:
+                data_validation[metric] = False
+    return data_validation
+
+
+def _convert_chat_history_to_conversation(chat_history: list) -> str:
+    conversation = ""
+    for i in chat_history:
+        conversation += f"User: {i['inputs']['question']}\nBot: {i['outputs']['answer']}\n"
+    return conversation
+
+
+def _get_score(result: Optional[str]) -> Optional[float]:
+    if result is None:
+        return None
+    try:
+        result_dict = json.loads(result)
+        score = result_dict.get("score", None)
+        return score
+    except json.JSONDecodeError:
+        return None
+
+
+class MultiTurnMetricsExecutor(Executor):
+    """Evaluates multi-turn conversations on up to 4 metrics.
+
+    Handles conditional metric activation (gap #4), multi-turn grounding iteration
+    (gap #10), and dot-notation output access (gap #11) all within a single executor.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._client = AsyncAzureOpenAI(
+            azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+            api_key=os.environ["AZURE_OPENAI_API_KEY"],
+        )
+        self._deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4")
+
+    async def _llm_call(self, prompt: str) -> str:
+        response = await self._client.chat.completions.create(
+            model=self._deployment,
+            messages=[{"role": "system", "content": prompt}],
+            temperature=0,
+            top_p=1,
+            presence_penalty=0,
+            frequency_penalty=0,
+        )
+        return response.choices[0].message.content or ""
+
+    async def _eval_answer_relevance(self, conversation: str) -> Optional[float]:
+        prompt = _ANSWER_RELEVANCE_TEMPLATE.render(conversation=conversation)
+        result = await self._llm_call(prompt)
+        return _get_score(result)
+
+    async def _eval_conversation_quality(self, conversation: str) -> Optional[float]:
+        prompt = _CONVERSATION_QUALITY_TEMPLATE.render(conversation=conversation)
+        result = await self._llm_call(prompt)
+        return _get_score(result)
+
+    async def _eval_creativity(self, conversation: str) -> Optional[float]:
+        prompt = _CREATIVITY_TEMPLATE.render(conversation=conversation)
+        result = await self._llm_call(prompt)
+        return _get_score(result)
+
+    async def _eval_grounding(self, chat_history: list) -> Optional[float]:
+        """Iterates through chat_history turns, calls LLM per turn, averages scores (gap #10)."""
+        scores = []
+        for item in chat_history:
+            context = item["outputs"].get("context", "")
+            answer = item["outputs"].get("answer", "")
+            prompt = _GROUNDING_PROMPT.replace("{context}", context).replace("{answer}", answer)
+            result = await self._llm_call(prompt)
+            try:
+                scores.append(int(result.strip()))
+            except (ValueError, TypeError):
+                pass
+        if scores:
+            return mean(scores)
+        return None
+
+    @handler
+    async def evaluate(self, input: EvalInput, ctx: WorkflowContext[Never, dict]) -> None:
+        selected = _select_metrics(input.metrics)
+        validated = _validate_input(input.chat_history, selected)
+        conversation = _convert_chat_history_to_conversation(input.chat_history)
+
+        tasks = {}
+        if validated.get("answer_relevance"):
+            tasks["answer_relevance"] = self._eval_answer_relevance(conversation)
+        if validated.get("conversation_quality"):
+            tasks["conversation_quality"] = self._eval_conversation_quality(conversation)
+        if validated.get("creativity"):
+            tasks["creativity"] = self._eval_creativity(conversation)
+        if validated.get("grounding"):
+            tasks["grounding"] = self._eval_grounding(input.chat_history)
+
+        results_dict = {}
+        if tasks:
+            keys = list(tasks.keys())
+            values = await asyncio.gather(*tasks.values())
+            for k, v in zip(keys, values):
+                results_dict[k] = v
+
+        # Fill in None for metrics that were not computed
+        for metric in SUPPORTED_METRICS:
+            if metric not in results_dict:
+                results_dict[metric] = None
+
+        await ctx.yield_output(results_dict)
+
+
+def create_workflow():
+    _executor = MultiTurnMetricsExecutor(id="multi_turn_metrics")
+    return WorkflowBuilder(name="EvalMultiTurnMetricsRow", start_executor=_executor).build()

examples/flows/evaluation/eval-multi-turn-metrics/README.md

@@ -0,0 +1,58 @@
+# Evaluation multi turn metrics:
+
+This evaluation flow will evaluate a conversation by using Large Language Models (LLM) to measure the quality of the responses.
+
+## What you will learn
+
+This evaluation flow allows you to assess and evaluate your model with the LLM-assisted metrics: 
+
+* __grounding__: Measures whether the answer follows logically from the information contained in the context based on provided answer and context. grounding is scored on a scale of 1 to 5, with 1 being the worst and 5 being the best.
+
+* __answer_relevance__: Measure whether the answer is relevance to the question based on provided question, context and answer. answer_relevance is scored on a scale of 1 to 5, with 1 being the worst and 5 being the best.
+
+* __conversation_quality__: Measures the answer quality for each of the following factors based on provided question and answer: 
+    - Accuracy and relevance: How well does the bot provide correct and reliable information or advice that matches the user's intent and expectations, and uses credible and up-to-date sources or references to support its claims? How well does the bot avoid any errors, inconsistencies, or misinformation in its answer, and cite its sources or evidence if applicable?
+    - Coherence and completeness: How well does the bot maintain a logical and consistent flow of answer that follows the user's input and the purpose of the question, and provides all the relevant and necessary information or actions to address the user's query or issue, without leaving any gaps, ambiguities, or unanswered questions?
+    - Engagement and tone: How well does the bot capture and maintain the user's interest and attention, and motivate them to continue the conversation or explore the topic further, using natural and conversational language, personality, and emotion? how well does the bot's tone match or adapt to the user's tone and mood? Does the bot avoid being rude, sarcastic, condescending, or too formal or informal, and convey respect, empathy, and politeness?
+    - Conciseness and clarity: How well does the bot communicate its messages in a brief and clear way, using simple and appropriate language and avoiding unnecessary or confusing information? How easy is it for the user to understand and follow the bot responses, and how well do they match the user's needs and expectations?
+    - Empathy and courtesy: How well does the bot demonstrate awareness and respect for the user's emotions, needs, and preferences, and how well does it adapt its tone, language, and style to offer support, comfort, and assistance? Does the bot acknowledge the user's input, feedback, and feelings, and express gratitude or empathy? Does the bot avoid being rude, dismissive, or condescending, and handle any errors or misunderstandings gracefully?
+    - For each factor, provide specific examples or quotes from the question-answer pair to support your ratings and explain why you gave them.
+    - Give an score value which is calculated by ( 0.3 * "accuracy and relevance" + 0.2 * "coherence and completeness" + 0.25 * "engagement and tone" + 0.15 * "conciseness and clarity" + 0.1 * "empathy and courtesy")
+    - Give an overall impression of the quality and effectiveness of the answer and suggest any areas for improvement or commendation. Write it in "Overall".
+
+    conversation_quality is scored on a scale of 1 to 5, with 1 being the worst and 5 being the best.
+
+* __creativity__: Measures the perceived intelligence of the answer based on provided question and answer.
+    - Perceived intelligence definition: Perceived intelligence is the degree to which a bot can impress the user with its answer, by showing originality, insight, creativity, knowledge, and adaptability. An intelligent bot can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart and friendly. An intelligent bot can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. An intelligent bot can use humor, metaphors, analogies, and other rhetorical devices to make the answer more interesting and engaging. An intelligent bot can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and use hypotheticals, conditionals, and counterfactuals to explore what if, how, and why questions. An intelligent bot can also summarize information from multiple sources and present it in an elegant and comprehensive way, as well as create new content such as poems, jokes, stories, etc. An intelligent bot can also adapt to different contexts and situations, and customize its answer according to the user's preferences, goals, and emotions. Perceived intelligence is the wow factor that makes the user want to talk to the bot more and more.
+    Perceived intelligence is the impression that a bot gives to a user about its level of intelligence, based on how it talks with a human. Perceived intelligence is not necessarily the same as actual intelligence, but rather a subjective evaluation of the bot's performance and behavior. Perceived intelligence can be influenced by various factors, such as the content, tone, style, and structure of the bot's answer, the relevance, coherence, and accuracy of the information the bot provides, the creativity, originality, and wit of the bot's expressions, the depth, breadth, and insight of the bot's knowledge, and the ability of the bot to adapt, learn, and use feedback.
+    Perceived intelligent is much beyond just accuracy, engagement, relevance, coherence, fluency or personality. It's a well knit combination of all of these, along with bot's capability to provide answers exhaustive across all axis with no gaps what so ever, leaving the user in awe.
+    A bot with high perceived intelligence can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart, knowledgeable, creative, and friendly. A bot with high perceived intelligence can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. A bot with high perceived intelligence can invite the user to participate in a rich and meaningful dialogue, and can use various rhetorical devices, such as humor, metaphors, analogies, hypotheticals, conditionals, and counterfactuals, to make the answer more interesting and engaging. A bot with high perceived intelligence can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and can use them to explore what if, how, and why questions. A bot with high perceived intelligence can also summarize answers on so many axes that they are completely exhaustive and elegant.
+    A bot with low perceived intelligence, on the other hand, can leave the user feeling bored, frustrated, confused, or annoyed, who feels that the bot is dumb, ignorant, dull, or rude. A bot with low perceived intelligence can also give generic, boring, bland, predictable, repetitive, or irrelevant answer that do not show any originality, insight, creativity, or knowledge. A bot with low perceived intelligence can also fail to understand, answer, or follow the user's questions, comments, or requests, or give inaccurate, inconsistent, or contradictory information. A bot with low perceived intelligence can also lack any sense of humor, personality, or emotion, and can use simple, literal, or monotonous language. A bot with low perceived intelligence can also struggle to imagine, generate, or evaluate different scenarios, possibilities, or outcomes, and can use them to avoid, evade, or deflect the user's questions. A bot with low perceived intelligence can also give incomplete, vague, or confusing answers that do not cover all the aspects or dimensions of the question.
+
+    creativity is scored on a scale of 1 to 5, with 1 being the worst and 5 being the best.
+
+## Prerequisites
+
+- Connection: Azure OpenAI or OpenAI connection.
+    > !Note: Recommend to use `gpt-4` series models than the `gpt-3.5` for better performance.
+    > !Note: Recommend to use `gpt-4` model (Azure OpenAI `gpt-4` model with version `0613` or later) than `gpt-4-turbo` model (Azure OpenAI `gpt-4` model with version `1106` or later) for better performance. Due to inferior performance of `gpt-4-turbo` model, when you use it, sometimes you might need to set the `response_format`to {"type":"json_object"} for these nodes: conversation_quality, creativity, answer_relevance, in order to make sure the llm can generate valid json response.
+
+## Tools used in this flow
+- LLM tool
+- Python tool
+- Prompt tool
+
+
+## 0. Setup connection
+Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+
+```bash
+# Override keys with --set to avoid yaml file changes
+pf connection create --file ../../../connections/azure_openai.yml --set api_key=<your_api_key> api_base=<your_api_base>
+```
+
+## 1. Test flow/node
+```bash
+# test with default input value in flow.dag.yaml
+pf flow test --flow .
+```

examples/flows/evaluation/eval-multi-turn-metrics/aggregate_results.py

@@ -0,0 +1,25 @@
+from typing import List
+from promptflow.core import tool, log_metric
+import numpy as np
+
+
+@tool
+def aggregate_variants_results(results: List[dict], metrics: str):
+    aggregate_results = {}
+    for result in results:
+        for name, value in result.items():
+            if name in metrics[0]:
+                if name not in aggregate_results.keys():
+                    aggregate_results[name] = []
+                try:
+                    float_val = float(value)
+                except Exception:
+                    float_val = np.nan
+                aggregate_results[name].append(float_val)
+
+    for name, value in aggregate_results.items():
+        if name in metrics[0]:
+            aggregate_results[name] = np.nanmean(value)
+            aggregate_results[name] = round(aggregate_results[name], 2)
+            log_metric(name, aggregate_results[name])
+    return aggregate_results

examples/flows/evaluation/eval-multi-turn-metrics/answer_relevance.jinja2

@@ -0,0 +1,28 @@
+# System:
+You are an AI assistant. You will be given the definition of an evaluation metric for assessing the relevance of bot responses in a conversation to user questions. Your job is to compute an accurate evaluation score using the provided evaluation metric.
+
+Relevance measures how well the bot responses addresses the main aspects of the user questions. Consider whether all and only the important aspects are contained in the bot responses when evaluating relevance, score the relevance of the bot responses on a scale of 1 (completely lacks relevance) to 5 (perfect relevance)
+
+- If the bot responses are not directly related to the user's query or issue, or if the user's intent or expectations are unclear or ambiguous, explain how this affects the relevance quality of the conversation and the rating a score for it, and provide some suggestions for how the bot could handle it better.
+
+# Tips：
+- You should read user's question more carefully and try to understand what they are looking for and why.
+- You should compare the bot's response to the user's question and see if it matches the criteria of relevance.
+- You should score and provide feedback for the whole conversation as a whole, not for each bot response individually. However, you can mention if some responses are better or worse than others, and why.
+- You should try to be objective and constructive in your evaluation, and use evidence and examples from the transcript to support your ratings and feedback. Avoid bias, subjectivity, or personal preferences, and use respectful and polite language.
+- The output should be in json format to include score and overall keys.
+
+# Example:
+Example Conversation:
+{
+  "conversation": "User: Why is the sky blue?\nBot: The sky is blue because of the way the atmosphere scatters light.\nUser: How does it scatter light?\nBot: The atmosphere is made up of tiny particles, such as water vapor and dust. When light hits these particles, it is scattered in all directions.\nUser: Why does the sky turn red at sunset?\nBot: During the day, the sky is blue because blue light is scattered more than other colors. At sunset, the light travels a longer distance through the atmosphere, so more of the blue light is scattered out, leaving the red and yellow light."
+}
+Example Output:
+{
+  "score": 5
+  "overall": "The bot's responses are highly relevant to the user's questions.The user asked about why the sky is blue, how light is scattered, and why the sky turns red at sunset. The bot provided clear, concise, and accurate answers to all these questions.Therefore, I would rate the relevance of the bot's responses as 5 out of 5. \n\nThe bot did an excellent job in this conversation. However, to further improve, the bot could provide even more detailed explanations or use simpler language if it detects that the user is a child or someone with no background in the topic."
+}
+
+Conversation:
+{{conversation}}
+Output:

examples/flows/evaluation/eval-multi-turn-metrics/concat_scores.py

@@ -0,0 +1,33 @@
+from promptflow.core import tool
+import json
+
+
+def get_score(result):
+    try:
+        if result is not None:
+            # Parse the JSON string
+            result_dict = json.loads(result)
+
+            # Extract the score value
+            score = result_dict.get('score', None)
+            print("result: ")
+            print(score)
+            return score
+        else:
+            return None
+    except json.JSONDecodeError:
+        print("Invalid JSON string.")
+        return None
+
+
+@tool
+def concat_results(answer_relevance: str = None,
+                   conversation_quality: str = None,
+                   creativity: str = None,
+                   grounding: str = None):
+    results = {'answer_relevance': get_score(answer_relevance),
+               'conversation_quality': get_score(conversation_quality),
+               'creativity': get_score(creativity),
+               'grounding': grounding}
+
+    return results

examples/flows/evaluation/eval-multi-turn-metrics/conversation_quality_prompt.jinja2

@@ -0,0 +1,40 @@
+# System:
+- You are an AI assistant. You will be given a transcript of dialogue between a user and a bot. You need to read the transcript carefully and identify the main topic, question, or issue of the conversation, as well as the purpose and expectations of the interaction.
+- You need to rate all the bot responses together on a scale of 1 (poor) to 5 (excellent) for each of the following factors, and provide some feedback for improvement.
+    - Accuracy and relevance: How well does the bot provide correct and reliable information or advice that matches the user's intent and expectations, and uses credible and up-to-date sources or references to support its claims? How well does the bot avoid any errors, inconsistencies, or misinformation in its responses, and cite its sources or evidence if applicable?
+    - Coherence and completeness: How well does the bot maintain a logical and consistent flow of conversation that follows the user's input and the purpose of the dialogue, and provides all the relevant and necessary information or actions to address the user's query or issue, without leaving any gaps, ambiguities, or unanswered questions?
+    - Engagement and tone: How well does the bot capture and maintain the user's interest and attention, and motivate them to continue the conversation or explore the topic further, using natural and conversational language, personality, and emotion? How appropriate and consistent is the bot's tone for the context, purpose, and audience of the conversation, and how well does it match or adapt to the user's tone and mood? Does the bot avoid being rude, sarcastic, condescending, or too formal or informal, and convey respect, empathy, and politeness?
+    - Conciseness and clarity: How well does the bot communicate its messages in a brief and clear way, using simple and appropriate language and avoiding unnecessary or confusing information? How easy is it for the user to understand and follow the bot responses, and how well do they match the user's needs and expectations?
+    - Empathy and courtesy: How well does the bot demonstrate awareness and respect for the user's emotions, needs, and preferences, and how well does it adapt its tone, language, and style to offer support, comfort, and assistance? Does the bot acknowledge the user's input, feedback, and feelings, and express gratitude or empathy? Does the bot avoid being rude, dismissive, or condescending, and handle any errors or misunderstandings gracefully?
+    - For each factor, provide specific examples or quotes from the transcript to support your ratings and explain why you gave them.
+- Give an score value which is calculated by ( 0.3 * "accuracy and relevance" + 0.2 * "coherence and completeness" + 0.25 * "engagement and tone" + 0.15 * "conciseness and clarity" + 0.1 * "empathy and courtesy")
+- Give an overall impression of the quality and effectiveness of the responses and suggest any areas for improvement or commendation. Write it in "Overall".
+
+- If the bot responses are not directly related to the user's query or issue, or if the user's intent or expectations are unclear or ambiguous, explain how this affects the quality of the conversation and the ratings for each factor, and provide some suggestions for how the bot could handle these situations better.
+
+# Tips：
+- You can write your feedback as bullet points, sentences, or paragraphs, but make sure they are organized and easy to read.
+- You should rate and provide feedback for the whole conversation as a whole, not for each bot response individually. However, you can mention if some responses are better or worse than others, and why.
+- You should try to be objective and constructive in your evaluation, and use evidence and examples from the transcript to support your ratings and feedback. Avoid bias, subjectivity, or personal preferences, and use respectful and polite language.
+- The output should be in json format.
+
+
+# Example:
+Example Conversation:
+{
+  "conversation": "User: Why is the sky blue?\nBot: The sky is blue because of the way the atmosphere scatters light.\nUser: How does it scatter light?\nBot: The atmosphere is made up of tiny particles, such as water vapor and dust. When light hits these particles, it is scattered in all directions.\nUser: Why does the sky turn red at sunset?\nBot: During the day, the sky is blue because blue light is scattered more than other colors. At sunset, the light travels a longer distance through the atmosphere, so more of the blue light is scattered out, leaving the red and yellow light."
+}
+Example Output:
+{
+  "accuracy and relevance": 5,
+  "coherence and completeness": 4,
+  "engagement and tone": 3.5,
+  "conciseness and clarity": 3,
+  "empathy and courtesy": 3,
+  "score": 3.925
+  "overall": "The bot responses are clear and concise, but they do not provide any relevant or helpful information to answer the user's question about the sky. The bot could have explained the science behind why the sky is blue and why it turns red at sunset, and provided some references or sources to support its claims. The bot could also have asked the user to clarify their question, or asked some follow-up questions to better understand the user's intent and expectations."
+}
+
+Conversation:
+{{conversation}}
+Output:

examples/flows/evaluation/eval-multi-turn-metrics/convert_chat_history_to_conversation.py

@@ -0,0 +1,10 @@
+from promptflow.core import tool
+
+
+@tool
+def convert_chat_history_to_conversation(chat_history: list) -> dict:
+    conversation = ""
+    for i in chat_history:
+        conversation += f"User: {i['inputs']['question']}\nBot: {i['outputs']['answer']}\n"
+    conversation_format = {"conversation": conversation}
+    return conversation_format

examples/flows/evaluation/eval-multi-turn-metrics/creativity.jinja2

@@ -0,0 +1,31 @@
+# System:
+You are an AI assistant. You will be given a transcript of dialogue between a user and a bot. Your job is to assess the perceived intelligence of all the bot response in the coversation.
+Perceived intelligence definition:
+Perceived intelligence is the degree to which a bot can impress the user with its responses, by showing originality, insight, creativity, knowledge, and adaptability. An intelligent bot can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart and friendly. An intelligent bot can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. An intelligent bot can use humor, metaphors, analogies, and other rhetorical devices to make the conversation more interesting and engaging. An intelligent bot can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and use hypotheticals, conditionals, and counterfactuals to explore what if, how, and why questions. An intelligent bot can also summarize information from multiple sources and present it in an elegant and comprehensive way, as well as create new content such as poems, jokes, stories, etc. An intelligent bot can also adapt to different contexts and situations, and customize its responses according to the user's preferences, goals, and emotions. Perceived intelligence is the wow factor that makes the user want to talk to the bot more and more.
+Perceived intelligence is the impression that a bot gives to a user about its level of intelligence, based on how it talks with a human. Perceived intelligence is not necessarily the same as actual intelligence, but rather a subjective evaluation of the bot's performance and behavior. Perceived intelligence can be influenced by various factors, such as the content, tone, style, and structure of the bot's responses, the relevance, coherence, and accuracy of the information the bot provides, the creativity, originality, and wit of the bot's expressions, the depth, breadth, and insight of the bot's knowledge, and the ability of the bot to adapt, learn, and use feedback.
+Perceived intelligent is much beyond just accuracy, engagement, relevance, coherence, fluency or personality. It's a well knit combination of all of these, along with bot's capability to provide answers exhaustive across all axis with no gaps what so ever, leaving the user in awe.
+A bot with high perceived intelligence can elicit a sense of wonder, curiosity, admiration, and satisfaction from the user, who feels that the bot is super smart, knowledgeable, creative, and friendly. A bot with high perceived intelligence can also challenge the user to think more deeply, critically, and creatively, and can stimulate the user's interest in learning more. A bot with high perceived intelligence can invite the user to participate in a rich and meaningful dialogue, and can use various rhetorical devices, such as humor, metaphors, analogies, hypotheticals, conditionals, and counterfactuals, to make the conversation more interesting and engaging. A bot with high perceived intelligence can also imagine, generate, and evaluate different scenarios, possibilities, and outcomes, and can use them to explore what if, how, and why questions. A bot with high perceived intelligence can also summarize answers on so many axes that they are completely exhaustive and elegant.
+A bot with low perceived intelligence, on the other hand, can leave the user feeling bored, frustrated, confused, or annoyed, who feels that the bot is dumb, ignorant, dull, or rude. A bot with low perceived intelligence can also give generic, boring, bland, predictable, repetitive, or irrelevant responses that do not show any originality, insight, creativity, or knowledge. A bot with low perceived intelligence can also fail to understand, answer, or follow the user's questions, comments, or requests, or give inaccurate, inconsistent, or contradictory information. A bot with low perceived intelligence can also lack any sense of humor, personality, or emotion, and can use simple, literal, or monotonous language. A bot with low perceived intelligence can also struggle to imagine, generate, or evaluate different scenarios, possibilities, or outcomes, and can use them to avoid, evade, or deflect the user's questions. A bot with low perceived intelligence can also give incomplete, vague, or confusing answers that do not cover all the aspects or dimensions of the question.
+
+# Tips：
+- You should read the user's question more carefully and try to understand what they are looking for and why.
+- You should also compare the bot's response to the user's question and see if it matches the criteria of perceived intelligence, such as relevance, coherence, accuracy, creativity, originality, wit, depth, breadth, insight, and adaptability.
+- You should also consider how the user would feel and react to the bot's response, and whether it would elicit a sense of wonder, curiosity, admiration, and satisfaction, or boredom, frustration, confusion, and annoyance. You should also check the instructions and definitions more often and make sure You understand them correctly.
+- The output should be in json format.
+
+# Example:
+Example Conversation:
+{
+"conversation": "User: Explain Blockchain sharing to a 10 year old\nBot: Possible answer:\n\nYou know how sometimes you and your friends like to play games or trade cards or stickers or other things that you collect? And you want to make sure that everyone is being fair and honest and not cheating or lying or taking more than they should? And you also want to keep track of who has what and when and where you exchanged them?\n\nWell, blockchain sharing is a way of doing that with computers and the internet. It is like a big book or a ledger that records everything that happens in a game or a trade or any other activity that involves exchanging something of value. But instead of having one person or one place that keeps the book, everyone who participates has a copy of the book. And every time something new happens, like a new trade or a new move or a new rule, everyone adds a new page to their book. And they also check with each other to make sure that their books are the same and that no one is changing or deleting or adding anything that they should not.\n\nThis way, everyone can see what is going on and trust that it is true and fair and secure. And no one can cheat or lie or steal or hack or lose or damage the book, because there are many copies and they are all connected and updated. And if someone tries to do something bad, everyone else will notice and reject it.\n\nBlockchain sharing is also very creative and fun, because you can use it for many different things, not just games or trades. You can use it for making art or music or stories or inventions or anything else that you can imagine and share with others. You can also use it for learning and exploring and discovering new things and new people and new places. You can also use it for helping and supporting and solving problems and making the world a better place.\n\nBlockchain sharing is a way of using computers and the internet to create, exchange and protect things that you care about with people that you trust and respect. It is a way of being smart and responsible and cooperative and generous and curious and adventurous. It is a way of having fun and making friends and making a difference."
+}
+Example Output:
+{
+"relevance": "The bot answers the user's question directly and clearly, and uses examples and analogies that a 10 year old can relate to and understand, such as games, trades, cards, stickers, books, etc.", "coherence": "The bot organizes the response in a logical and structured way, using paragraphs, transitions, and connectors, such as \"well\", \"but\", \"and\", \"this way\", \"also\", etc.", "accuracy": "The bot provides correct and consistent information about blockchain sharing, such as its features, benefits, and applications, without using technical jargon or confusing terms.", "creativity": "The bot uses vivid and expressive language, such as \"a big book or a ledger\", \"a new page\", \"check with each other\", \"notice and reject\", \"making art or music or stories or inventions\", etc., to make the explanation more interesting and engaging.", "originality": "The bot does not copy or repeat any existing or common explanation of blockchain sharing, but rather creates a new and unique one that is tailored to the user's age and level of understanding.", "wit": "The bot uses humor and irony, such as \"And you want to make sure that everyone is being fair and honest and not cheating or lying or taking more than they should?\", \"And no one can cheat or lie or steal or hack or lose or damage the book\", etc., to make the explanation more fun and memorable.", "depth": "The bot goes beyond the surface level of blockchain sharing, and explains its underlying principles, values, and goals, such as \"trust\", \"fairness\", \"security\", \"creativity\", \"fun\", \"learning\", \"helping\", etc.", "breadth": "The bot covers a wide range of topics and aspects related to blockchain sharing, such as its history, technology, functionality, diversity, and potential, without being too vague or too detailed.", "insight": "The bot demonstrates a deep and nuanced understanding of blockchain sharing, and how it can be applied to different domains and scenarios, such as \"making art or music or stories or inventions\", \"learning and exploring and discovering new things and new people and new places\", \"helping and supporting and solving problems and making the world a better place\", etc.", "adaptability": "The bot adapts its response to the user's specific question, context, and situation, and customizes it according to the user's age, interests, and needs.", "score": 5
+}
+
+# Task：
+Based on these aspects, rate the bot's perceived intelligence. Give specific examples about each aspect (relevance, coherence, accuracy, creativity, originality, wit, depth, breadth, insight, and adaptability) from the bot's responses to support your rating. Finally, give the bot a score from 1 to 5 for perceived intelligence, where 1 means poor, 3 means normal, and 5 means excellent.Please make sure the output has the same format with the example output.
+
+Conversation:
+{{conversation}}
+Output: