chore: import upstream snapshot with attribution

examples/flows/evaluation/eval-summarization-maf/.env.example

@@ -0,0 +1,4 @@
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key
+AZURE_OPENAI_API_VERSION=2024-02-01
+AZURE_OPENAI_DEPLOYMENT=gpt-4

examples/flows/evaluation/eval-summarization-maf/aggregation.py

@@ -0,0 +1,19 @@
+from typing import Dict, List
+
+
+def aggregate(
+    coherence: List[float],
+    consistency: List[float],
+    fluency: List[float],
+    relevance: List[float],
+) -> Dict[str, float]:
+    average_fluency = sum(fluency) / len(fluency) if fluency else 0.0
+    average_consistency = sum(consistency) / len(consistency) if consistency else 0.0
+    average_relevance = sum(relevance) / len(relevance) if relevance else 0.0
+    average_coherence = sum(coherence) / len(coherence) if coherence else 0.0
+    return {
+        "average_fluency": average_fluency,
+        "average_consistency": average_consistency,
+        "average_relevance": average_relevance,
+        "average_coherence": average_coherence,
+    }

examples/flows/evaluation/eval-summarization-maf/data.jsonl

@@ -0,0 +1,2 @@
+{"document": "this is a test document", "summary": "test document"}
+{"document": "this is a test document2", "summary": "test document2"}

examples/flows/evaluation/eval-summarization-maf/eval_runner.py

@@ -0,0 +1,54 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+@dataclass
+class EvalResult:
+    per_row_outputs: List[Any]
+    metrics: Dict[str, Any]
+    errors: List[tuple] = field(default_factory=list)
+
+
+class EvalRunner:
+    def __init__(self, workflow_factory: Callable[[], Any], aggregate_fn: Callable[..., dict],
+                 concurrency: int = 5, input_mapping: Optional[Dict[str, str]] = None):
+        self._workflow_factory = workflow_factory
+        self._aggregate_fn = aggregate_fn
+        self._concurrency = concurrency
+        self._input_mapping = input_mapping
+
+    async def run(self, dataset: List[Any]) -> EvalResult:
+        semaphore = asyncio.Semaphore(self._concurrency)
+        per_row_outputs: List[Any] = [None] * len(dataset)
+        errors: List[tuple] = []
+
+        async def _run_row(index: int, row: Any) -> None:
+            async with semaphore:
+                wf = self._workflow_factory()
+                result = await wf.run(row)
+                per_row_outputs[index] = result.get_outputs()[0]
+
+        tasks = [_run_row(i, row) for i, row in enumerate(dataset)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        succeeded_outputs: List[Any] = []
+        for i, r in enumerate(results):
+            if isinstance(r, Exception):
+                errors.append((i, r))
+            else:
+                succeeded_outputs.append(per_row_outputs[i])
+
+        aggregation_inputs = self._transpose(succeeded_outputs)
+        if self._input_mapping:
+            aggregation_inputs = {self._input_mapping.get(k, k): v for k, v in aggregation_inputs.items()}
+        metrics = self._aggregate_fn(**aggregation_inputs)
+        return EvalResult(per_row_outputs=succeeded_outputs, metrics=metrics, errors=errors)
+
+    @staticmethod
+    def _transpose(outputs: List[Any]) -> Dict[str, Any]:
+        if not outputs:
+            return {"values": []}
+        if not isinstance(outputs[0], dict):
+            return {"values": outputs}
+        keys = outputs[0].keys()
+        return {k: [o[k] for o in outputs] for k in keys}

examples/flows/evaluation/eval-summarization-maf/prompts/coherence.jinja2

@@ -0,0 +1,32 @@
+You will be given one summary written for a source document.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+Evaluation Criteria:
+
+Coherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby "the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic."
+
+Evaluation Steps:
+
+1. Read the source document carefully and identify the main topic and key points.
+2. Read the summary and compare it to the source document. Check if the summary covers the main topic and key points of the source document, and if it presents them in a clear and logical order.
+3. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria.
+
+
+Example:
+
+
+Source Document:
+
+{{Document}}
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Coherence:

examples/flows/evaluation/eval-summarization-maf/prompts/consistency.jinja2

@@ -0,0 +1,33 @@
+You will be given a source document. You will then be given one summary written for this source document.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+
+Evaluation Criteria:
+
+Consistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. 
+
+Evaluation Steps:
+
+1. Read the source document carefully and identify the main facts and details it presents.
+2. Read the summary and compare it to the source document. Check if the summary contains any factual errors that are not supported by the source document.
+3. Assign a score for consistency based on the Evaluation Criteria.
+
+
+Example:
+
+
+Source Document: 
+
+{{Document}}
+
+Summary: 
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Consistency:

examples/flows/evaluation/eval-summarization-maf/prompts/fluency.jinja2

@@ -0,0 +1,26 @@
+You will be given one summary written for a source document.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+
+Evaluation Criteria:
+
+Fluency (1-3): the quality of the summary in terms of grammar, spelling, punctuation, word choice, and sentence structure.
+
+- 1: Poor. The summary has many errors that make it hard to understand or sound unnatural.
+- 2: Fair. The summary has some errors that affect the clarity or smoothness of the text, but the main points are still comprehensible.
+- 3: Good. The summary has few or no errors and is easy to read and follow.
+
+
+Example:
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Fluency (1-3):

examples/flows/evaluation/eval-summarization-maf/prompts/relevance.jinja2

@@ -0,0 +1,33 @@
+You will be given one summary written for a source document.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+Evaluation Criteria:
+
+Relevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.
+
+Evaluation Steps:
+
+1. Read the summary and the source document carefully.
+2. Compare the summary to the source document and identify the main points of the source document.
+3. Assess how well the summary covers the main points of the source document, and how much irrelevant or redundant information it contains.
+4. Assign a relevance score from 1 to 5.
+
+
+Example:
+
+
+Source Document:
+
+{{Document}}
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Relevance:

examples/flows/evaluation/eval-summarization-maf/requirements.txt

@@ -0,0 +1,5 @@
+agent-framework>=1.0.1
+openai
+python-dotenv
+jinja2
+tenacity

examples/flows/evaluation/eval-summarization-maf/run_eval.py

@@ -0,0 +1,44 @@
+import argparse
+import asyncio
+import json
+from pathlib import Path
+from aggregation import aggregate
+from eval_runner import EvalRunner
+from workflow import EvalInput, create_workflow
+
+DEFAULT_DATA = Path(__file__).parent / "data.jsonl"
+
+
+def load_dataset(path: Path) -> list[EvalInput]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                rows.append(EvalInput(document=obj["document"], summary=obj["summary"]))
+    return rows
+
+
+async def main(data_path: Path, concurrency: int):
+    dataset = load_dataset(data_path)
+    print(f"Loaded {len(dataset)} rows from {data_path}")
+
+    runner = EvalRunner(
+        workflow_factory=create_workflow,
+        aggregate_fn=aggregate,
+        concurrency=concurrency,
+    )
+    result = await runner.run(dataset)
+
+    print("\n--- Metrics ---")
+    for key, value in result.metrics.items():
+        print(f"  {key}: {value}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data", type=Path, default=DEFAULT_DATA)
+    parser.add_argument("--concurrency", type=int, default=5)
+    args = parser.parse_args()
+    asyncio.run(main(args.data, args.concurrency))

examples/flows/evaluation/eval-summarization-maf/test_eval.py

@@ -0,0 +1,45 @@
+import asyncio
+import json
+from pathlib import Path
+from workflow import EvalInput, create_workflow
+
+
+async def test_single_row():
+    """Smoke test - requires Azure OpenAI credentials in .env"""
+    wf = create_workflow()
+    result = await wf.run(EvalInput(
+        document="The quick brown fox jumps over the lazy dog.",
+        summary="A fox jumped over a dog.",
+    ))
+    scores = result.get_outputs()[0]
+    assert isinstance(scores, dict)
+    assert set(scores.keys()) == {"coherence", "consistency", "fluency", "relevance"}
+    for v in scores.values():
+        assert isinstance(v, float)
+    print(f"PASS: test_single_row (scores={scores})")
+
+
+async def test_data_jsonl():
+    """Run eval on every row in data.jsonl"""
+    data_path = Path(__file__).parent / "data.jsonl"
+    rows = [json.loads(line) for line in data_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    wf = create_workflow()
+    for i, row in enumerate(rows):
+        result = await wf.run(EvalInput(
+            document=row["document"],
+            summary=row["summary"],
+        ))
+        scores = result.get_outputs()[0]
+        assert isinstance(scores, dict), f"Row {i}: expected dict, got {type(scores)}"
+        print(f"  Row {i}: scores={scores}")
+    print(f"PASS: test_data_jsonl ({len(rows)} rows)")
+
+
+async def main():
+    await test_single_row()
+    await test_data_jsonl()
+    print("\nAll tests passed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/flows/evaluation/eval-summarization-maf/workflow.py

@@ -0,0 +1,141 @@
+import asyncio
+import logging
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+from dotenv import load_dotenv
+from jinja2 import Template
+from openai import AsyncAzureOpenAI
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from typing_extensions import Never
+from agent_framework import Executor, WorkflowBuilder, WorkflowContext, handler
+
+load_dotenv()
+
+logger = logging.getLogger(__name__)
+
+# Load Jinja2 prompt templates (gap #3: type:prompt as Jinja2 render)
+_PROMPTS_DIR = Path(__file__).parent / "prompts"
+_COHERENCE_TEMPLATE = Template(_PROMPTS_DIR.joinpath("coherence.jinja2").read_text(encoding="utf-8"))
+_CONSISTENCY_TEMPLATE = Template(_PROMPTS_DIR.joinpath("consistency.jinja2").read_text(encoding="utf-8"))
+_FLUENCY_TEMPLATE = Template(_PROMPTS_DIR.joinpath("fluency.jinja2").read_text(encoding="utf-8"))
+_RELEVANCE_TEMPLATE = Template(_PROMPTS_DIR.joinpath("relevance.jinja2").read_text(encoding="utf-8"))
+
+# Dimension configs: (template, max_score, needs_document)
+_DIMENSIONS = {
+    "coherence": (_COHERENCE_TEMPLATE, 5, True),
+    "consistency": (_CONSISTENCY_TEMPLATE, 5, True),
+    "fluency": (_FLUENCY_TEMPLATE, 3, False),
+    "relevance": (_RELEVANCE_TEMPLATE, 5, True),
+}
+
+
+@dataclass
+class EvalInput:
+    document: str
+    summary: str
+
+
+def _parse_output(output: str, max_score: float) -> float:
+    matched = re.findall(r"(?<!\S)\d+(?:\.\d+)?", output)
+    if matched:
+        if len(matched) == 1:
+            score = float(matched[0])
+            if score > max_score:
+                raise ValueError(f"Parsed number: {score} was larger than max score: {max_score}")
+        else:
+            raise ValueError(f"More than one number detected in input: {output}")
+    else:
+        raise ValueError(f'No number detected in input: "{output}"')
+    return score
+
+
+def _aggregate_llm_scores(llm_responses: List[str], max_score: float) -> float:
+    all_scores = []
+    error_count = 0
+    for generated in llm_responses:
+        try:
+            parsed = _parse_output(generated, max_score)
+            all_scores.append(parsed)
+        except ValueError as e:
+            logger.warning(e)
+            error_count += 1
+    if error_count:
+        logger.warning(f"{error_count} out of {len(llm_responses)} scores were discarded")
+    if not all_scores:
+        return 0.0
+    return sum(all_scores) / len(all_scores)
+
+
+class SummarizationGEvalExecutor(Executor):
+    """Runs G-Eval (n=20 sampling) for all 4 summarization dimensions.
+
+    Uses raw openai SDK because MAF Agent doesn't support n>1 (gap #9).
+    Uses AzureOpenAI client directly (gap #6: AzureOpenAIConnection in Python).
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._client = AsyncAzureOpenAI(
+            azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+            api_key=os.environ["AZURE_OPENAI_API_KEY"],
+        )
+        self._deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-4")
+
+    @retry(wait=wait_random_exponential(multiplier=1, min=1, max=120), stop=stop_after_attempt(10), reraise=True)
+    async def _call_geval(self, prompt: str) -> list:
+        response = await self._client.chat.completions.create(
+            model=self._deployment,
+            messages=[{"role": "system", "content": prompt}],
+            temperature=2,
+            max_tokens=5,
+            top_p=1,
+            frequency_penalty=0,
+            presence_penalty=0,
+            n=20,
+        )
+        responses = []
+        for choice in response.choices:
+            try:
+                responses.append(choice.message.content)
+            except (KeyError, AttributeError):
+                pass
+        return responses
+
+    async def _score_dimension(self, name: str, document: str, summary: str) -> float:
+        template, max_score, needs_doc = _DIMENSIONS[name]
+        if needs_doc:
+            prompt = template.render(Document=document, Summary=summary)
+        else:
+            prompt = template.render(Summary=summary)
+        responses = await self._call_geval(prompt)
+        return _aggregate_llm_scores(responses, max_score)
+
+    @handler
+    async def evaluate(self, input: EvalInput, ctx: WorkflowContext[Never, dict]) -> None:
+        # Run all 4 dimensions concurrently
+        results = await asyncio.gather(
+            self._score_dimension("coherence", input.document, input.summary),
+            self._score_dimension("consistency", input.document, input.summary),
+            self._score_dimension("fluency", input.document, input.summary),
+            self._score_dimension("relevance", input.document, input.summary),
+        )
+        await ctx.yield_output({
+            "coherence": results[0],
+            "consistency": results[1],
+            "fluency": results[2],
+            "relevance": results[3],
+        })
+
+
+def create_workflow():
+    _geval = SummarizationGEvalExecutor(id="geval_summarization")
+    return WorkflowBuilder(name="EvalSummarizationRow", start_executor=_geval).build()