chore: import upstream snapshot with attribution

examples/flows/evaluation/eval-basic-maf/aggregation.py

@@ -0,0 +1,23 @@
+"""
+Aggregation function for eval-basic.
+
+Extracted from the original `aggregate.py` PromptFlow node.
+- `@tool` decorator removed (not needed in MAF).
+- `log_metric()` replaced with returning metrics as a dict.
+"""
+
+from typing import Dict, List
+
+
+def aggregate(processed_results: List[str]) -> Dict[str, int]:
+    """Aggregate per-row results into batch-level metrics.
+
+    :param processed_results: List of "Correct"/"Incorrect" strings from all rows.
+    :returns: Dict with metric name → value.
+    """
+    results_num = len(processed_results)
+    correct_num = processed_results.count("Correct")
+    return {
+        "results_num": results_num,
+        "correct_num": correct_num,
+    }

examples/flows/evaluation/eval-basic-maf/data.jsonl

@@ -0,0 +1 @@
+{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow."}

examples/flows/evaluation/eval-basic-maf/eval_runner.py

@@ -0,0 +1,116 @@
+"""
+EvalRunner — batch evaluation orchestrator for MAF workflows.
+
+Bridges the gap between MAF's single-invocation workflow model and PromptFlow's
+batch-level `aggregation: true` pattern.
+
+Usage:
+    runner = EvalRunner(workflow, aggregate_fn, input_mapping={"values": "processed_results"})
+    result = await runner.run(dataset)
+    print(result.metrics)
+"""
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """Result of a batch evaluation run."""
+
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    """Runs a MAF workflow per row, collects outputs, then calls an aggregation function.
+
+    This mirrors PromptFlow's two-phase execution model:
+      Phase 1 — run each row through the workflow concurrently
+      Phase 2 — pass all collected outputs to the aggregation function
+
+    MAF workflows do not support concurrent execution on a single instance,
+    so `workflow_factory` creates a fresh workflow for each concurrent row.
+
+    :param workflow_factory: A zero-arg callable that returns a built MAF workflow.
+    :param aggregate_fn: A function that receives collected outputs and returns a metrics dict.
+    :param concurrency: Max concurrent workflow.run() calls (prevents rate-limit errors).
+    :param input_mapping: Optional rename map for transposed keys → aggregation function params.
+        For single-value outputs, _transpose produces {"values": [...]}.  If the aggregation
+        function expects a different param name (e.g., "processed_results"), pass
+        {"values": "processed_results"}.
+    """
+
+    def __init__(
+        self,
+        workflow_factory: Callable[[], Any],
+        aggregate_fn: Callable[..., dict],
+        concurrency: int = 5,
+        input_mapping: Optional[Dict[str, str]] = None,
+    ):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        """Execute the full eval pipeline: per-row → collect → aggregate.
+
+        :param dataset: List of inputs to pass to workflow.run() (one per row).
+        :returns: EvalResult with per-row outputs, metrics, and any errors.
+        """
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        # Phase 1: run all rows concurrently (bounded by semaphore)
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Separate successes from failures
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        # Transpose outputs into aggregation inputs
+        aggregation_inputs = self._transpose(succeeded_outputs)
+
+        # Apply parameter name mapping if provided
+        if self._input_mapping:
+            aggregation_inputs = {
+                self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
+            }
+
+        # Phase 2: call aggregation function
+        metrics = self._aggregate_fn(**aggregation_inputs)
+
+        return EvalResult(
+            per_row_outputs=succeeded_outputs,
+            metrics=metrics,
+            errors=errors,
+        )
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        """Transpose per-row outputs into aggregation-ready keyword args.
+
+        - If outputs are plain values (str, int, float): {"values": [v1, v2, ...]}
+        - If outputs are dicts: {key: [row1[key], row2[key], ...]} for each key
+        """
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-basic-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-basic-maf/run_eval.py

@@ -0,0 +1,70 @@
+"""
+Entry point: run the eval-basic evaluation as a batch.
+
+Loads the test dataset, runs the per-row workflow for each row,
+then aggregates results.
+
+Usage:
+    python run_eval.py
+    python run_eval.py --data path/to/data.jsonl --concurrency 10
+"""
+
+import argparse
+import asyncio
+import json
+from pathlib import Path
+
+from aggregation import aggregate
+from eval_runner import EvalResult, EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    """Load a JSONL file into a list of EvalInput."""
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=obj["groundtruth"], prediction=obj["prediction"]))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int) -> EvalResult:
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "processed_results"},
+    )
+
+    result = await runner.run(dataset)
+
+    print("\n--- Per-row outputs ---")
+    for i, output in enumerate(result.per_row_outputs):
+        print(f"  Row {i}: {output}")
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+    if result.errors:
+        print(f"\n--- Errors ({len(result.errors)}) ---")
+        for idx, err in result.errors:
+            print(f"  Row {idx}: {err}")
+
+    return result
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run eval-basic evaluation batch")
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA, help="Path to JSONL dataset")
+    parser.add_argument("--concurrency", type=int, default=5, help="Max concurrent workflow runs")
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-basic-maf/test_eval_basic.py

@@ -0,0 +1,100 @@
+"""
+Test script for the eval-basic MAF conversion.
+
+Verifies both the per-row workflow and the full batch evaluation pipeline.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Test the per-row workflow with a matching pair."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="Sunny"))
+    output = result.get_outputs()[0]
+    assert output == "Correct", f"Expected 'Correct', got '{output}'"
+    print("PASS: test_single_row")
+
+
+async def test_single_row_mismatch():
+    """Test the per-row workflow with a non-matching pair."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="rainy"))
+    output = result.get_outputs()[0]
+    assert output == "Incorrect", f"Expected 'Incorrect', got '{output}'"
+    print("PASS: test_single_row_mismatch")
+
+
+async def test_batch_evaluation():
+    """Test the full batch pipeline with EvalRunner."""
+    dataset = [
+        EvalInput(groundtruth="APP", prediction="APP"),
+        EvalInput(groundtruth="APP", prediction="WEB"),
+        EvalInput(groundtruth="DB", prediction="db"),
+    ]
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+
+    assert result.per_row_outputs == ["Correct", "Incorrect", "Correct"], (
+        f"Unexpected per-row outputs: {result.per_row_outputs}"
+    )
+    assert result.metrics["results_num"] == 3, f"Expected 3 results, got {result.metrics['results_num']}"
+    assert result.metrics["correct_num"] == 2, f"Expected 2 correct, got {result.metrics['correct_num']}"
+    assert len(result.errors) == 0, f"Unexpected errors: {result.errors}"
+    print("PASS: test_batch_evaluation")
+
+
+async def test_empty_dataset():
+    """Test with an empty dataset."""
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run([])
+    assert result.per_row_outputs == []
+    assert result.metrics["results_num"] == 0
+    assert len(result.errors) == 0
+    print("PASS: test_empty_dataset")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["prediction"],
+        ))
+        grade = result.get_outputs()[0]
+        assert grade in ("Correct", "Incorrect"), f"Row {i}: unexpected grade '{grade}'"
+        print(f"  Row {i}: grade={grade}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_single_row_mismatch()
+    await test_batch_evaluation()
+    await test_empty_dataset()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-basic-maf/workflow.py

@@ -0,0 +1,56 @@
+"""
+Per-row MAF workflow for eval-basic.
+
+Converts the `line_process` node from the original PromptFlow evaluation flow.
+Each workflow invocation processes a single (groundtruth, prediction) pair and
+yields "Correct" or "Incorrect".
+
+Original flow graph (per-row nodes only):
+    [inputs: groundtruth, prediction] → [line_process] → output
+"""
+
+import asyncio
+from dataclasses import dataclass
+
+from typing_extensions import Never
+
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class LineProcessExecutor(Executor):
+    """Replaces the `line_process` Python node.
+
+    Compares groundtruth and prediction (case-insensitive) and yields the result.
+    """
+
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, str]) -> None:
+        result = "Correct" if input.groundtruth.lower() == input.prediction.lower() else "Incorrect"
+        await ctx.yield_output(result)
+
+
+def create_workflow():
+    """Create a fresh workflow instance.
+
+    MAF workflows do not support concurrent execution, so each batch row
+    needs its own workflow instance.
+    """
+    _line_process = LineProcessExecutor(id="line_process")
+    return WorkflowBuilder(name="EvalBasicRow", start_executor=_line_process).build()
+
+
+async def main():
+    """Quick smoke test with a single row."""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="sunny", prediction="Sunny"))
+    print(result.get_outputs()[0])  # → "Correct"
+
+
+if __name__ == "__main__":
+    asyncio.run(main())