chore: import upstream snapshot with attribution

examples/chartqa/multimodal_utils.py

@@ -0,0 +1,113 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""
+Multimodal support utilities for Agent Lightning.
+
+This module provides helper functions for working with multimodal agents,
+particularly for vision-language tasks.
+"""
+
+from __future__ import annotations
+
+import base64
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Union
+
+import requests
+from PIL import Image
+from PIL.Image import Image as PILImage
+
+__all__ = [
+    "encode_image_to_base64",
+    "create_image_message",
+]
+
+
+def encode_image_to_base64(image: Union[str, Path, PILImage], max_size: int = 2048) -> str:
+    """
+    Encode an image to base64 string for multimodal LLM APIs.
+
+    Args:
+        image: Image source (file path, URL, or PIL Image object)
+        max_size: Maximum dimension for resizing
+
+    Returns:
+        Base64 encoded image string with data URI prefix
+
+    Raises:
+        ImportError: If PIL (Pillow) is not installed
+        TypeError: If image type is not supported
+
+    Examples:
+        >>> encoded = encode_image_to_base64("photo.jpg")
+        >>> encoded[:30]
+        'data:image/jpeg;base64,/9j/4A...'
+
+        >>> from PIL import Image
+        >>> img = Image.open("photo.jpg")
+        >>> encoded = encode_image_to_base64(img)
+    """
+    # Load image
+    if isinstance(image, (str, Path)):
+        image_str = str(image)
+        if image_str.startswith(("http://", "https://")):
+            response = requests.get(image_str, timeout=30)
+            response.raise_for_status()
+            img = Image.open(BytesIO(response.content))
+        else:
+            img = Image.open(image_str)
+    elif hasattr(image, "mode"):
+        # PIL Image object
+        img = image
+    else:
+        raise TypeError(f"Unsupported image type: {type(image)}")
+
+    # Convert to RGB
+    if img.mode == "RGBA":
+        background = Image.new("RGB", img.size, (255, 255, 255))
+        background.paste(img, mask=img.split()[3])
+        img = background
+    elif img.mode != "RGB":
+        img = img.convert("RGB")
+
+    # Resize if needed
+    if max(img.size) > max_size:
+        img.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
+
+    # Encode
+    buffered = BytesIO()
+    img.save(buffered, format="JPEG", quality=85)
+    img_str = base64.b64encode(buffered.getvalue()).decode()
+
+    return f"data:image/jpeg;base64,{img_str}"
+
+
+def create_image_message(text: str, image: Union[str, Path, PILImage], use_base64: bool = True) -> dict[str, Any]:
+    """
+    Create an OpenAI-compatible multimodal message.
+
+    Args:
+        text: The text prompt/question
+        image: Image source (path, URL, or PIL Image)
+        use_base64: If True, encode as base64; if False, use URL directly
+
+    Returns:
+        Message dict with role="user" and multimodal content
+
+    Examples:
+        >>> msg = create_image_message("What's in the image?", "photo.jpg")
+        >>> msg["role"]
+        'user'
+        >>> len(msg["content"])
+        2
+    """
+    content: list[dict[str, Any]] = [{"type": "text", "text": text}]
+
+    if isinstance(image, str) and image.startswith(("http://", "https://")) and not use_base64:
+        content.append({"type": "image_url", "image_url": {"url": image}})
+    else:
+        encoded = encode_image_to_base64(image)
+        content.append({"type": "image_url", "image_url": {"url": encoded}})
+
+    return {"role": "user", "content": content}