chore: import upstream snapshot with attribution

python/samples/02-agents/multimodal_input/README.md

@@ -0,0 +1,119 @@
+# Multimodal Input Examples
+
+This folder contains examples demonstrating how to send multimodal content (images, audio, PDF files) to AI agents using the Agent Framework.
+
+## Examples
+
+### OpenAI Chat Client
+
+- **File**: `openai_chat_multimodal.py`
+- **Description**: Shows how to send images, audio, and PDF files to OpenAI's Chat Completions API
+- **Supported formats**: PNG/JPEG images, WAV/MP3 audio, PDF documents
+
+### Azure OpenAI Chat Client
+
+- **File**: `azure_chat_multimodal.py`
+- **Description**: Shows how to send images to Azure OpenAI Chat Completions API
+- **Supported formats**: PNG/JPEG images (PDF files are NOT supported by Chat Completions API)
+
+### Azure OpenAI Responses Client
+
+- **File**: `azure_responses_multimodal.py`
+- **Description**: Shows how to send images and PDF files to Azure OpenAI Responses API
+- **Supported formats**: PNG/JPEG images, PDF documents (full multimodal support)
+
+## Environment Variables
+
+Set the following environment variables before running the examples:
+
+**For OpenAI:**
+- `OPENAI_API_KEY`: Your OpenAI API key
+
+**For Azure OpenAI:**
+
+- `AZURE_OPENAI_ENDPOINT`: Your Azure OpenAI endpoint
+- `AZURE_OPENAI_MODEL`: The name of your Azure OpenAI chat model deployment
+- `AZURE_OPENAI_MODEL`: The name of your Azure OpenAI responses model deployment
+
+Optionally for Azure OpenAI:
+- `AZURE_OPENAI_API_VERSION`: The API version to use (default is `2024-10-21`)
+- `AZURE_OPENAI_API_KEY`: Your Azure OpenAI API key (if not using `AzureCliCredential`)
+
+**Note:** You can also provide configuration directly in code instead of using environment variables:
+```python
+# Example: Pass the Foundry project endpoint directly
+client = FoundryChatClient(
+    credential=AzureCliCredential(),
+    project_endpoint="https://your-project.services.ai.azure.com",
+    model="your-deployment-name",
+)
+```
+
+## Authentication
+
+The Azure example uses `AzureCliCredential` for authentication. Run `az login` in your terminal before running the example, or replace `AzureCliCredential` with your preferred authentication method (e.g., provide `api_key` parameter).
+
+## Running the Examples
+
+```bash
+# Run OpenAI example
+python openai_chat_multimodal.py
+
+# Run Azure Chat example (requires az login or API key)
+python azure_chat_multimodal.py
+
+# Run Azure Responses example (requires az login or API key)
+python azure_responses_multimodal.py
+```
+
+## Using Your Own Files
+
+The examples include small embedded test files for demonstration. To use your own files:
+
+### Method 1: Data URIs (recommended)
+
+```python
+import base64
+
+# Load and encode your file
+with open("path/to/your/image.jpg", "rb") as f:
+    image_data = f.read()
+    image_base64 = base64.b64encode(image_data).decode('utf-8')
+    image_uri = f"data:image/jpeg;base64,{image_base64}"
+
+# Use in DataContent
+Content.from_uri(
+    uri=image_uri,
+    media_type="image/jpeg"
+)
+```
+
+### Method 2: Raw bytes
+
+```python
+# Load raw bytes
+with open("path/to/your/image.jpg", "rb") as f:
+    image_bytes = f.read()
+
+# Use in DataContent
+Content.from_data(
+    data=image_bytes,
+    media_type="image/jpeg"
+)
+```
+
+## Supported File Types
+
+| Type      | Formats              | Notes                          |
+| --------- | -------------------- | ------------------------------ |
+| Images    | PNG, JPEG, GIF, WebP | Most common image formats      |
+| Audio     | WAV, MP3             | For transcription and analysis |
+| Documents | PDF                  | Text extraction and analysis   |
+
+## API Differences
+
+- **OpenAI Chat Completions API**: Supports images, audio, and PDF files
+- **Azure OpenAI Chat Completions API**: Supports images only (no PDF/audio file types)
+- **Azure OpenAI Responses API**: Supports images and PDF files (full multimodal support)
+
+Choose the appropriate client based on your multimodal needs and available APIs.

python/samples/02-agents/multimodal_input/azure_chat_multimodal.py

@@ -0,0 +1,48 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+
+from agent_framework import Content, Message
+from agent_framework.foundry import FoundryChatClient
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+
+def create_sample_image() -> str:
+    """Create a simple 1x1 pixel PNG image for testing."""
+    # This is a tiny yellow pixel in PNG format
+    png_data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+    return f"data:image/png;base64,{png_data}"
+
+
+async def test_image() -> None:
+    """Test image analysis with Azure OpenAI."""
+    # For authentication, run `az login` command in terminal or replace AzureCliCredential with preferred
+    # authentication option. Requires AZURE_OPENAI_ENDPOINT and FOUNDRY_MODEL
+    # environment variables to be set.
+    # Alternatively, you can pass model explicitly:
+    # client = FoundryChatClient(credential=AzureCliCredential(), model="your-deployment-name")
+    client = FoundryChatClient(credential=AzureCliCredential())
+    image_uri = create_sample_image()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What's in this image?"),
+            Content.from_uri(uri=image_uri, media_type="image/png"),
+        ],
+    )
+    response = await client.get_response([message])
+    print(f"Image Response: {response}")
+
+
+async def main() -> None:
+    print("=== Testing Azure OpenAI Multimodal ===")
+    print("Testing image analysis (supported by Chat Completions API)")
+    await test_image()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/samples/02-agents/multimodal_input/azure_responses_multimodal.py

@@ -0,0 +1,81 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+from pathlib import Path
+
+from agent_framework import Content, Message
+from agent_framework.foundry import FoundryChatClient
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+ASSETS_DIR = Path(__file__).resolve().parents[2] / "shared" / "sample_assets"
+
+
+def load_sample_pdf() -> bytes:
+    """Read the bundled sample PDF for tests."""
+    pdf_path = ASSETS_DIR / "sample.pdf"
+    return pdf_path.read_bytes()
+
+
+def create_sample_image() -> str:
+    """Create a simple 1x1 pixel PNG image for testing."""
+    # This is a tiny yellow pixel in PNG format
+    png_data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+    return f"data:image/png;base64,{png_data}"
+
+
+async def test_image() -> None:
+    """Test image analysis with Azure OpenAI Responses API."""
+    # For authentication, run `az login` command in terminal or replace AzureCliCredential with preferred
+    # authentication option. Requires AZURE_OPENAI_ENDPOINT and FOUNDRY_MODEL
+    # environment variables to be set.
+    # Alternatively, you can pass model explicitly:
+    # client = FoundryChatClient(credential=AzureCliCredential(), model="your-deployment-name")
+    client = FoundryChatClient(credential=AzureCliCredential())
+
+    image_uri = create_sample_image()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What's in this image?"),
+            Content.from_uri(uri=image_uri, media_type="image/png"),
+        ],
+    )
+
+    response = await client.get_response([message])
+    print(f"Image Response: {response}")
+
+
+async def test_pdf() -> None:
+    """Test PDF document analysis with Azure OpenAI Responses API."""
+    client = FoundryChatClient(credential=AzureCliCredential())
+
+    pdf_bytes = load_sample_pdf()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What information can you extract from this document?"),
+            Content.from_data(
+                data=pdf_bytes,
+                media_type="application/pdf",
+                additional_properties={"filename": "sample.pdf"},
+            ),
+        ],
+    )
+
+    response = await client.get_response([message])
+    print(f"PDF Response: {response}")
+
+
+async def main() -> None:
+    print("=== Testing Azure OpenAI Responses API Multimodal ===")
+    print("The Responses API supports both images AND PDFs")
+    await test_image()
+    await test_pdf()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/samples/02-agents/multimodal_input/openai_chat_multimodal.py

@@ -0,0 +1,115 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+import base64
+import struct
+from pathlib import Path
+
+from agent_framework import Content, Message
+from agent_framework.openai import OpenAIChatClient, OpenAIChatCompletionClient
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+ASSETS_DIR = Path(__file__).resolve().parents[2] / "shared" / "sample_assets"
+
+"""
+Leverage multimodel capabilities of different models.
+
+Uses the OpenAIChatClient and OpenAIChatCompletionClient to demonstrate multimodal input handling with the gpt-4o and gpt-4o-audio-preview models, respectively. The sample includes demonstrations for image, audio, and PDF inputs, showcasing how to create appropriate Content objects and send them in messages to the chat clients.
+
+"""
+
+
+def load_sample_pdf() -> bytes:
+    """Read the bundled sample PDF for tests."""
+    pdf_path = ASSETS_DIR / "sample.pdf"
+    return pdf_path.read_bytes()
+
+
+def create_sample_image() -> str:
+    """Create a simple 1x1 pixel PNG image for testing."""
+    # This is a tiny yellow pixel in PNG format
+    png_data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+    return f"data:image/png;base64,{png_data}"
+
+
+def create_sample_audio() -> str:
+    """Create a minimal WAV file for testing (0.1 seconds of silence)."""
+    wav_header = (
+        b"RIFF"
+        + struct.pack("<I", 44)  # file size
+        + b"WAVEfmt "
+        + struct.pack("<I", 16)  # fmt chunk
+        + struct.pack("<HHIIHH", 1, 1, 8000, 16000, 2, 16)  # PCM, mono, 8kHz
+        + b"data"
+        + struct.pack("<I", 1600)  # data chunk
+        + b"\x00" * 1600  # 0.1 sec silence
+    )
+    audio_b64 = base64.b64encode(wav_header).decode()
+    return f"data:audio/wav;base64,{audio_b64}"
+
+
+async def test_image() -> None:
+    """Test image analysis with OpenAI."""
+    client = OpenAIChatClient(model="gpt-4o")
+
+    image_uri = create_sample_image()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What's in this image?"),
+            Content.from_uri(uri=image_uri, media_type="image/png"),
+        ],
+    )
+
+    response = await client.get_response([message])
+    print(f"Image Response: {response}")
+
+
+async def test_audio() -> None:
+    """Test audio analysis with OpenAI."""
+    client = OpenAIChatCompletionClient(model="gpt-4o-audio-preview-2025-06-03")
+
+    audio_uri = create_sample_audio()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What do you hear in this audio?"),
+            Content.from_uri(uri=audio_uri, media_type="audio/wav"),
+        ],
+    )
+
+    response = await client.get_response([message])
+    print(f"Audio Response: {response}")
+
+
+async def test_pdf() -> None:
+    """Test PDF document analysis with OpenAI."""
+    client = OpenAIChatClient(model="gpt-4o")
+
+    pdf_bytes = load_sample_pdf()
+    message = Message(
+        role="user",
+        contents=[
+            Content.from_text(text="What information can you extract from this document?"),
+            Content.from_data(
+                data=pdf_bytes, media_type="application/pdf", additional_properties={"filename": "employee_report.pdf"}
+            ),
+        ],
+    )
+
+    response = await client.get_response([message])
+    print(f"PDF Response: {response}")
+
+
+async def main() -> None:
+    print("=== Testing OpenAI Multimodal ===")
+    await test_image()
+    await test_audio()
+    await test_pdf()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())