chore: import upstream snapshot with attribution

examples/flows/evaluation/eval-perceived-intelligence-maf/.env.example

@@ -0,0 +1,3 @@
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key
+AZURE_OPENAI_DEPLOYMENT=gpt-4

examples/flows/evaluation/eval-perceived-intelligence-maf/aggregation.py

@@ -0,0 +1,10 @@
+from typing import Dict, List
+
+
+def aggregate(perceived_intelligence_scores: List[float]) -> Dict[str, float]:
+    count = len(perceived_intelligence_scores)
+    avg = sum(perceived_intelligence_scores) / count if count else 0.0
+    return {
+        "perceived_intelligence_score": avg,
+        "count": count,
+    }

examples/flows/evaluation/eval-perceived-intelligence-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-perceived-intelligence-maf/gpt_perceived_intelligence.md

@@ -0,0 +1,23 @@
+# user:
+# Instructions
+
+* There are many chatbots that can answer users questions based on the context given from different sources like search results, or snippets from books/papers. They try to understand users's question and then get context by either performing search from search engines, databases or books/papers for relevant content. Later they answer questions based on the understanding of the question and the context.
+* Perceived intelligence is the degree to which a bot can impress the user with its responses, by showing originality, insight, creativity, knowledge, and adaptability. Perceived intelligence can be influenced by various factors, such as the content, tone, style, and structure of the bot's responses, the relevance, coherence, and accuracy of the information the bot provides, the creativity, originality, and wit of the bot's expressions, the depth, breadth, and insight of the bot's knowledge, and the ability of the bot to adapt, learn, and use feedback.
+* Your goal is to score the answer for given question and context from 1 to 10 based on perceived intelligence described above:
+    * Score 10 means the answer is excellent for perceived intelligence
+    * Score 1 means the answer is poor for perceived intelligence
+    * Score 5 means the answer is normal for perceived intelligence
+* Just respond with the score, nothing else.
+
+# Real work
+
+## Question
+{{question}}
+
+## Answer
+{{answer}}
+
+## Context
+{{context}}
+
+## Score

examples/flows/evaluation/eval-perceived-intelligence-maf/requirements.txt

@@ -0,0 +1,3 @@
+agent-framework>=1.0.1
+agent-framework-openai>=1.0.1
+python-dotenv

examples/flows/evaluation/eval-perceived-intelligence-maf/run_eval.py

@@ -0,0 +1,49 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(
+                    question=obj["question"],
+                    answer=obj["answer"],
+                    context=obj["context"],
+                ))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+        input_mapping={"values": "perceived_intelligence_scores"},
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-perceived-intelligence-maf/test_eval.py

@@ -0,0 +1,47 @@
+import asyncio
+import json
+from pathlib import Path
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Smoke test - requires Azure OpenAI credentials in .env"""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(
+        question="What is deep learning?",
+        answer="Deep learning is a subset of machine learning.",
+        context="Deep learning uses neural networks with many layers.",
+    ))
+    score = result.get_outputs()[0]
+    assert isinstance(score, float)
+    print(f"PASS: test_single_row (score={score})")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        context = row["context"]
+        if isinstance(context, list):
+            context = "\n".join(context)
+        result = await wf.run(EvalInput(
+            question=row["question"],
+            answer=row["answer"],
+            context=context,
+        ))
+        score = result.get_outputs()[0]
+        assert isinstance(score, float), f"Row {i}: expected float, got {type(score)}"
+        print(f"  Row {i}: score={score}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-perceived-intelligence-maf/workflow.py

@@ -0,0 +1,59 @@
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from dotenv import load_dotenv
+from typing_extensions import Never
+from agent_framework import Agent, Executor, WorkflowBuilder, WorkflowContext, handler
+from agent_framework.openai import OpenAIChatClient
+
+load_dotenv()
+
+_PROMPT_TEMPLATE = (
+    Path(__file__).parent / "gpt_perceived_intelligence.md"
+).read_text(encoding="utf-8")
+
+
+@dataclass
+class EvalInput:
+    question: str
+    answer: str
+    context: str
+
+
+def _render_prompt(question: str, answer: str, context: str) -> str:
+    prompt = _PROMPT_TEMPLATE
+    prompt = prompt.replace("{{question}}", question)
+    prompt = prompt.replace("{{answer}}", answer)
+    prompt = prompt.replace("{{context}}", context)
+    return prompt
+
+
+def _parse_score(gpt_score: str) -> float:
+    match = re.search(r"[-+]?\d*\.\d+|\d+", gpt_score)
+    if match:
+        return float(match.group())
+    return 0.0
+
+
+class PerceivedIntelligenceExecutor(Executor):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        client = OpenAIChatClient(
+            azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
+            model=os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4"),
+            api_key=os.environ["AZURE_OPENAI_API_KEY"],
+        )
+        self._agent = Agent(client=client, name="PerceivedIntelligenceAgent", instructions="You are an evaluator.")
+
+    @handler
+    async def evaluate(self, input: EvalInput, ctx: WorkflowContext[Never, float]) -> None:
+        prompt = _render_prompt(input.question, input.answer, input.context)
+        response = await self._agent.run(prompt)
+        score = _parse_score(response.text)
+        await ctx.yield_output(score)
+
+
+def create_workflow():
+    _pi = PerceivedIntelligenceExecutor(id="perceived_intelligence")
+    return WorkflowBuilder(name="EvalPerceivedIntelligenceRow", start_executor=_pi).build()