chore: import upstream snapshot with attribution

python/samples/02-agents/evaluation/evaluate_agent.py

@@ -0,0 +1,81 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Evaluate an agent with local checks — no API keys needed.
+
+Demonstrates the simplest evaluation workflow:
+1. Define checks using the @evaluator decorator
+2. Run evaluate_agent() which calls agent.run() under the covers
+3. Assert results in CI or inspect interactively
+
+Usage:
+    uv run python samples/02-agents/evaluation/evaluate_agent.py
+"""
+
+import asyncio
+import os
+
+from agent_framework import (
+    Agent,
+    LocalEvaluator,
+    evaluate_agent,
+    evaluator,
+    keyword_check,
+)
+from agent_framework.foundry import FoundryChatClient
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+# A custom check — parameter names determine what data you receive
+@evaluator
+def is_helpful(response: str) -> bool:
+    """Check the response isn't empty or a refusal."""
+    refusals = ["i can't", "i'm not able", "i don't know"]
+    return len(response) > 10 and not any(r in response.lower() for r in refusals)
+
+
+async def main() -> None:
+    client = FoundryChatClient(
+        project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
+        model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"),
+        credential=AzureCliCredential(),
+    )
+
+    agent = Agent(
+        client=client,
+        name="weather-assistant",
+        instructions="You are a helpful weather assistant.",
+    )
+
+    # Combine built-in and custom checks
+    local = LocalEvaluator(
+        keyword_check("weather"),  # response must mention "weather"
+        is_helpful,  # custom check
+    )
+
+    # evaluate_agent() calls agent.run() for each query, then evaluates
+    results = await evaluate_agent(
+        agent=agent,
+        queries=[
+            "What's the weather like in Seattle?",
+            "Will it rain in London tomorrow?",
+            "What should I wear for 30°C weather?",
+        ],
+        evaluators=local,
+    )
+
+    for r in results:
+        print(f"{r.provider}: {r.passed}/{r.total} passed")
+        for item in r.items:
+            print(f"  [{item.status}] Q: {(item.input_text or '')[:50]}  A: {(item.output_text or '')[:50]}...")
+            for score in item.scores:
+                print(f"    {'PASS' if score.passed else 'FAIL'} {score.name}")
+
+    # Use in CI: will raise EvalNotPassedError if any check fails
+    # results[0].raise_for_status()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/samples/02-agents/evaluation/evaluate_multimodal.py

@@ -0,0 +1,121 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Evaluate multimodal (image) conversations locally.
+
+Demonstrates that the evaluation pipeline preserves image content:
+1. Build EvalItems with image content in conversations
+2. Use @evaluator checks that inspect multimodal content
+3. Verify images flow through the eval pipeline intact
+
+Usage:
+    uv run python samples/02-agents/evaluation/evaluate_multimodal.py
+"""
+
+import asyncio
+import base64
+
+from agent_framework import (
+    Content,
+    EvalItem,
+    LocalEvaluator,
+    Message,
+    evaluator,
+)
+
+# -- Custom evaluators that inspect multimodal content --
+
+
+@evaluator
+def has_image_content(conversation: list) -> bool:
+    """Check that the conversation contains at least one image."""
+    return any(
+        c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/")
+        for m in conversation
+        for c in (m.contents or [])
+    )
+
+
+@evaluator
+def response_describes_image(response: str) -> bool:
+    """Check that the assistant response acknowledges the image."""
+    image_words = {"image", "picture", "photo", "shows", "depicts", "see"}
+    return any(word in response.lower() for word in image_words)
+
+
+@evaluator
+def image_count(conversation: list) -> float:
+    """Return the number of images in the conversation as a score."""
+    count = sum(
+        1
+        for m in conversation
+        for c in (m.contents or [])
+        if c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/")
+    )
+    return float(count)
+
+
+# A tiny 1x1 red PNG for demonstration (no external dependencies needed)
+_TINY_PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+)
+
+
+async def main() -> None:
+    # Build eval items with multimodal content (no agent run needed)
+    items = [
+        # Item 1: User sends an image URL with a question
+        EvalItem(
+            conversation=[
+                Message(
+                    "user",
+                    [
+                        Content.from_text("What do you see in this image?"),
+                        Content.from_uri(
+                            "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png",
+                            media_type="image/png",
+                        ),
+                    ],
+                ),
+                Message("assistant", ["The image shows two dice on a transparent background."]),
+            ]
+        ),
+        # Item 2: User sends inline image bytes
+        EvalItem(
+            conversation=[
+                Message(
+                    "user",
+                    [
+                        Content.from_text("Describe this picture"),
+                        Content.from_data(data=_TINY_PNG, media_type="image/png"),
+                    ],
+                ),
+                Message("assistant", ["I see a small red image — it appears to be a single pixel."]),
+            ]
+        ),
+        # Item 3: Text-only conversation (should fail has_image_content)
+        EvalItem(
+            conversation=[
+                Message("user", ["Tell me about cats"]),
+                Message("assistant", ["Cats are wonderful pets."]),
+            ]
+        ),
+    ]
+
+    local = LocalEvaluator(
+        has_image_content,
+        response_describes_image,
+        image_count,
+    )
+
+    results = await local.evaluate(items)
+
+    print(f"\n{results.provider}: {results.passed}/{results.total} passed")
+    for item in results.items:
+        print(f"\n  [{item.status}] Q: {(item.input_text or '')[:60]}...")
+        for score in item.scores:
+            symbol = "PASS" if score.passed else "FAIL"
+            print(f"    {symbol} {score.name}: {score.score}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/samples/02-agents/evaluation/evaluate_with_expected.py

@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Evaluate an agent with expected outputs and tool call checks.
+
+Demonstrates ground-truth comparison and tool usage evaluation:
+1. Provide expected outputs alongside queries
+2. Use built-in tool_calls_present for tool verification
+3. Combine multiple evaluation criteria
+
+Usage:
+    uv run python samples/02-agents/evaluation/evaluate_with_expected.py
+"""
+
+import asyncio
+import os
+
+from agent_framework import (
+    Agent,
+    LocalEvaluator,
+    evaluate_agent,
+    evaluator,
+    tool_calls_present,
+)
+from agent_framework.foundry import FoundryChatClient
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+@evaluator
+def response_matches_expected(response: str, expected_output: str) -> float:
+    """Score based on word overlap with expected output."""
+    if not expected_output:
+        return 1.0
+    response_words = set(response.lower().split())
+    expected_words = set(expected_output.lower().split())
+    return len(response_words & expected_words) / max(len(expected_words), 1)
+
+
+async def main() -> None:
+    client = FoundryChatClient(
+        project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
+        model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"),
+        credential=AzureCliCredential(),
+    )
+
+    agent = Agent(
+        client=client,
+        name="math-tutor",
+        instructions="You are a math tutor. Answer concisely.",
+    )
+
+    local = LocalEvaluator(
+        response_matches_expected,
+        tool_calls_present,  # verifies expected tools were called
+    )
+
+    results = await evaluate_agent(
+        agent=agent,
+        queries=["What is 2 + 2?", "What is the square root of 144?"],
+        expected_output=["4", "12"],
+        evaluators=local,
+    )
+
+    for r in results:
+        print(f"{r.provider}: {r.passed}/{r.total} passed")
+        for item in r.items:
+            print(f"  [{item.status}] {item.input_text} -> {(item.output_text or '')[:80]}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())