chore: import upstream snapshot with attribution

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/aggregation.py

@@ -0,0 +1,24 @@
+from typing import Dict, List
+
+
+def aggregate(processed_results: List[int]) -> Dict[str, float]:
+    num_exception = 0
+    num_correct = 0
+
+    for i in range(len(processed_results)):
+        if processed_results[i] == -1:
+            num_exception += 1
+        elif processed_results[i] == 1:
+            num_correct += 1
+
+    num_total = len(processed_results)
+    accuracy = round(1.0 * num_correct / num_total, 2) if num_total else 0.0
+    error_rate = round(1.0 * num_exception / num_total, 2) if num_total else 0.0
+
+    return {
+        "num_total": num_total,
+        "num_correct": num_correct,
+        "num_exception": num_exception,
+        "accuracy": accuracy,
+        "error_rate": error_rate,
+    }

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/eval_runner.py

@@ -0,0 +1,70 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """Result of a batch evaluation run."""
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    """Runs a MAF workflow per row, collects outputs, then calls an aggregation function."""
+
+    def __init__(
+        self,
+        workflow_factory: Callable[[], Any],
+        aggregate_fn: Callable[..., dict],
+        concurrency: int = 5,
+        input_mapping: Optional[Dict[str, str]] = None,
+    ):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {
+                self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()
+            }
+
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(
+            per_row_outputs=succeeded_outputs,
+            metrics=metrics,
+            errors=errors,
+        )
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/requirements.txt

@@ -0,0 +1 @@
+agent-framework>=1.0.1

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/run_eval.py

@@ -0,0 +1,49 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "test_data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(groundtruth=str(obj["groundtruth"]), prediction=str(obj["prediction"])))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+    if result.errors:
+        print(f"\n--- Errors ({len(result.errors)}) ---")
+        for idx, err in result.errors:
+            print(f"  Row {idx}: {err}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/test_data.jsonl

@@ -0,0 +1,20 @@
+{"question": "What is the sum of 5 and 3?", "groundtruth": "8", "answer": "8"}
+{"question": "Subtract 7 from 10.", "groundtruth": "3", "answer": "3"}
+{"question": "Multiply 6 by 4.", "groundtruth": "24", "answer": "24"}
+{"question": "Divide 20 by 5.", "groundtruth": "4", "answer": "4"}
+{"question": "What is the square of 7?", "groundtruth": "49", "answer": "49"}
+{"question": "What is the square root of 81?", "groundtruth": "9", "answer": "9"}
+{"question": "If a rectangle has a length of 10 and width of 5, what is the area?", "groundtruth": "50", "answer": "50"}
+{"question": "A circle has a radius of 7, what is the area? (Use 3.14 for pi)", "groundtruth": "153.86", "answer": "153.871"}
+{"question": "Solve for x in the equation 2x + 3 = 9.", "groundtruth": "3", "answer": "3"}
+{"question": "What is the value of x if 5x = 25?", "groundtruth": "5", "answer": "5"}
+{"question": "A car travels 200 miles in 4 hours. What is the average speed of the car?", "groundtruth": "50", "answer": "50"}
+{"question": "A car travels at a speed of 60 mph. How long will it take to travel 180 miles?", "groundtruth": "3", "answer": "3"}
+{"question": "If a car travels at a speed of 40 mph for 2 hours, how far will it travel?","groundtruth": "80", "answer": "80"}
+{"question":"A rectangle has length = 10 cm and width = 5 cm. What is its area?", "groundtruth":"50", "answer": "50"}
+{"question":"A circle has radius = 7 cm. What is its circumference? (Use pi =3.14)", "groundtruth":"43.96", "answer": "43.959"}
+{"question":"A triangle has base =10 cm and height =5 cm. What is its area?", "groundtruth":"25", "answer": "25"}
+{"question":"What is the slope of the line that passes through (2,3) and (4,7)?", "groundtruth":"2", "answer": "2"}
+{"question":"The distance between A and B is 2000km, A is moving towards B with speed 80km/hour, meanwhile B is moving towards A with speed 120km/hour, how many hours later A and B can meet?", "groundtruth":"10", "answer": "10"}
+{"question":"The lengths of the two perpendicular sides of a right triangle are 6cm and 8cm. What is the length of the hypotenuse?", "groundtruth": "10", "answer": "10"}
+{"question":"A is running with average speed 10km/hour, A already run half hour. B start to chase A along the same route with average speed 15km/hour, how many hours B will take to meet A?", "groundtruth":"1", "answer": "2"}

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/test_eval.py

@@ -0,0 +1,75 @@
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_correct():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="1.0", prediction="1"))
+    assert result.get_outputs()[0] == 1
+    print("PASS: test_single_correct")
+
+
+async def test_single_incorrect():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="2.1", prediction="2.0"))
+    assert result.get_outputs()[0] == 0
+    print("PASS: test_single_incorrect")
+
+
+async def test_json_error():
+    wf = create_workflow()
+    result = await wf.run(EvalInput(groundtruth="1.0", prediction="JSONDecodeError"))
+    assert result.get_outputs()[0] == -1
+    print("PASS: test_json_error")
+
+
+async def test_batch():
+    dataset = [
+        EvalInput(groundtruth="1.0", prediction="1"),
+        EvalInput(groundtruth="3.14", prediction="3.1415926"),
+        EvalInput(groundtruth="1.0", prediction="JSONDecodeError"),
+    ]
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=5,
+        input_mapping={"values": "processed_results"},
+    )
+    result = await runner.run(dataset)
+    assert result.metrics["num_total"] == 3
+    assert result.metrics["num_correct"] == 2
+    assert result.metrics["num_exception"] == 1
+    print("PASS: test_batch")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in test_data.jsonl"""
+    data_path = Path(__file__).parent / "test_data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            groundtruth=row["groundtruth"],
+            prediction=row["answer"],
+        ))
+        score = result.get_outputs()[0]
+        assert isinstance(score, int), f"Row {i}: expected int, got {type(score)}"
+        print(f"  Row {i}: score={score}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_correct()
+    await test_single_incorrect()
+    await test_json_error()
+    await test_batch()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-accuracy-maths-to-code-maf/workflow.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+
+@dataclass
+class EvalInput:
+    groundtruth: str
+    prediction: str
+
+
+class LineProcessExecutor(Executor):
+    @handler
+    async def process(self, input: EvalInput, ctx: WorkflowContext[Never, int]) -> None:
+        processed_result = 0
+
+        if input.prediction == "JSONDecodeError" or input.prediction.startswith("Unknown Error:"):
+            await ctx.yield_output(-1)
+            return
+
+        try:
+            groundtruth = float(input.groundtruth)
+            prediction = float(input.prediction)
+        except ValueError:
+            await ctx.yield_output(-1)
+            return
+
+        if round(prediction, 2) == round(groundtruth, 2):
+            processed_result = 1
+
+        await ctx.yield_output(processed_result)
+
+
+def create_workflow():
+    _line_process = LineProcessExecutor(id="line_process")
+    return WorkflowBuilder(name="EvalAccuracyMathsRow", start_executor=_line_process).build()