chore: import upstream snapshot with attribution

2026-07-13 13:39:25 +08:00
commit db620d33df
5151 changed files with 925932 additions and 0 deletions
@@ -0,0 +1,33 @@
+# DevUI Multi-Modal Agent
+
+Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding.
+
+## Setup
+
+1. Set environment variables (or create a `.env` file in `python/`):
+   ```bash
+   FOUNDRY_PROJECT_ENDPOINT=https://your-project.api.azureml.ms
+   AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
+   AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
+   ```
+
+2. Log in with Azure CLI:
+   ```bash
+   az login
+   ```
+
+3. Run with DevUI:
+   ```bash
+   uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
+   ```
+
+4. Open the DevUI URL in your browser and start uploading files.
+
+## What You Can Do
+
+- **Upload PDFs** — including scanned/image-based PDFs that LLM vision struggles with
+- **Upload images** — handwritten notes, infographics, charts
+- **Upload audio** — meeting recordings, call center calls (transcription with speaker ID)
+- **Upload video** — product demos, training videos (frame extraction + transcription)
+- **Ask questions** across all uploaded documents
+- **Check status** — "which documents are ready?" uses the auto-registered `list_documents()` tool
@@ -0,0 +1,6 @@
+# Copyright (c) Microsoft. All rights reserved.
+"""DevUI Multi-Modal Agent with Azure Content Understanding."""
+
+from .agent import agent
+
+__all__ = ["agent"]
@@ -0,0 +1,66 @@
+# Copyright (c) Microsoft. All rights reserved.
+"""DevUI Multi-Modal Agent — file upload + CU-powered analysis.
+
+This agent uses Azure Content Understanding to analyze uploaded files
+(PDFs, scanned documents, handwritten images, audio recordings, video)
+and answer questions about them through the DevUI web interface.
+
+Unlike the standard azure_responses_agent which sends files directly to the LLM,
+this agent uses CU for structured extraction — superior for scanned PDFs,
+handwritten content, audio transcription, and video analysis.
+
+Required environment variables:
+  FOUNDRY_PROJECT_ENDPOINT                 — Azure AI Foundry project endpoint
+  FOUNDRY_MODEL                            — Model deployment name (e.g. gpt-4.1)
+  AZURE_CONTENTUNDERSTANDING_ENDPOINT      — CU endpoint URL
+
+Run with DevUI:
+  uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
+"""
+
+import os
+
+from agent_framework import Agent
+from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
+from azure.core.credentials import AzureKeyCredential
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- Auth ---
+_credential = AzureCliCredential()
+_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
+_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
+
+cu = ContentUnderstandingContextProvider(
+    endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
+    credential=_cu_credential,
+    # max_wait controls how long before_run() waits for CU analysis before
+    # deferring to background.  For interactive DevUI use, a short timeout
+    # (e.g. 5s) keeps the chat responsive — the agent tells the user the
+    # file is still being analyzed and resolves it on the next turn.
+    # Use max_wait=None to always wait for analysis to complete.
+    max_wait=5.0,
+)
+
+client = FoundryChatClient(
+    project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
+    model=os.environ["FOUNDRY_MODEL"],
+    credential=_credential,
+)
+
+agent = Agent(
+    client=client,
+    name="MultiModalDocAgent",
+    instructions=(
+        "You are a helpful document analysis assistant. "
+        "When a user uploads files, they are automatically analyzed using Azure Content Understanding. "
+        "Use list_documents() to check which documents are ready, pending, or failed "
+        "and to see which files are available for answering questions. "
+        "Tell the user if any documents are still being analyzed. "
+        "You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
+        "When answering, cite specific content from the documents."
+    ),
+    context_providers=[cu],
+)
@@ -0,0 +1,51 @@
+# DevUI File Search Agent
+
+Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + OpenAI file_search RAG.
+
+## How It Works
+
+1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
+2. **CU analyzes** the file — auto-selects the right analyzer per media type
+3. **Markdown extracted** by CU is uploaded to an OpenAI vector store
+4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
+5. **Ask questions** across all uploaded documents with token-efficient RAG
+
+## Setup
+
+1. Set environment variables (or create a `.env` file in `python/`):
+   ```bash
+   FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
+   AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
+   AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
+   ```
+
+2. Log in with Azure CLI:
+   ```bash
+   az login
+   ```
+
+3. Run with DevUI:
+   ```bash
+   devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
+   ```
+
+4. Open the DevUI URL in your browser and start uploading files.
+
+## Supported File Types
+
+| Type | Formats | CU Analyzer (auto-detected) |
+|------|---------|----------------------------|
+| Documents | PDF, DOCX, XLSX, PPTX, HTML, TXT, Markdown | `prebuilt-documentSearch` |
+| Images | JPEG, PNG, TIFF, BMP | `prebuilt-documentSearch` |
+| Audio | WAV, MP3, FLAC, OGG, M4A | `prebuilt-audioSearch` |
+| Video | MP4, MOV, AVI, WebM | `prebuilt-videoSearch` |
+
+## vs. devui_multimodal_agent
+
+| Feature | multimodal_agent | file_search_agent |
+|---------|-----------------|-------------------|
+| CU extraction | ✅ Full content injected | ✅ Content indexed in vector store |
+| RAG | ❌ | ✅ file_search retrieves top-k chunks |
+| Large docs (100+ pages) | ⚠️ May exceed context window | ✅ Token-efficient |
+| Multiple large files | ⚠️ Context overflow risk | ✅ All indexed, searchable |
+| Best for | Small docs, quick inspection | Large docs, multi-file Q&A |
@@ -0,0 +1,6 @@
+# Copyright (c) Microsoft. All rights reserved.
+"""DevUI Multi-Modal Agent with CU + file_search RAG."""
+
+from .agent import agent
+
+__all__ = ["agent"]
@@ -0,0 +1,105 @@
+# Copyright (c) Microsoft. All rights reserved.
+"""DevUI Multi-Modal Agent — CU extraction + file_search RAG.
+
+This agent combines Azure Content Understanding with OpenAI file_search
+for token-efficient RAG over large or multi-modal documents.
+
+Upload flow:
+  1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
+  2. Extracted markdown is auto-uploaded to an OpenAI vector store
+  3. file_search tool is registered so the LLM retrieves top-k chunks
+  4. Vector store is configured to auto-expire after inactivity
+
+This is ideal for large documents (100+ pages), long audio recordings,
+or multiple files in the same conversation where full-context injection
+would exceed the LLM's context window.
+
+Analyzer auto-detection:
+  When no analyzer_id is specified, the provider auto-selects the
+  appropriate CU analyzer based on media type:
+    - Documents/images → prebuilt-documentSearch
+    - Audio            → prebuilt-audioSearch
+    - Video            → prebuilt-videoSearch
+
+Required environment variables:
+  FOUNDRY_PROJECT_ENDPOINT                 — Azure AI Foundry project endpoint
+  FOUNDRY_MODEL                            — Model deployment name (e.g. gpt-4.1)
+  AZURE_CONTENTUNDERSTANDING_ENDPOINT      — CU endpoint URL
+
+Run with DevUI:
+  devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
+"""
+
+import os
+
+from agent_framework import Agent
+from agent_framework.foundry import (
+    ContentUnderstandingContextProvider,
+    FileSearchConfig,
+    FoundryChatClient,
+)
+from azure.ai.projects import AIProjectClient
+from azure.core.credentials import AzureKeyCredential
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- Auth ---
+_credential = AzureCliCredential()
+_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
+_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
+
+_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
+
+# --- LLM client + sync vector store setup ---
+# DevUI loads agent modules synchronously at startup while an event loop is already
+# running, so we cannot use async APIs here. A sync AIProjectClient is used for
+# one-time vector store creation; runtime file uploads use client.client (async).
+client = FoundryChatClient(
+    project_endpoint=_endpoint,
+    model=os.environ["FOUNDRY_MODEL"],
+    credential=_credential,
+)
+
+_sync_project = AIProjectClient(endpoint=_endpoint, credential=_credential)  # type: ignore[arg-type]
+_sync_openai = _sync_project.get_openai_client()
+_vector_store = _sync_openai.vector_stores.create(
+    name="devui_cu_file_search",
+    expires_after={"anchor": "last_active_at", "days": 1},
+)
+_sync_openai.close()
+
+_file_search_tool = client.get_file_search_tool(
+    vector_store_ids=[_vector_store.id],
+    max_num_results=3,  # limit chunks to reduce input token usage
+)
+
+# --- CU context provider with file_search ---
+# client.client is the async OpenAI client used for runtime file uploads.
+# No analyzer_id → auto-selects per media type (documents, audio, video)
+cu = ContentUnderstandingContextProvider(
+    endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
+    credential=_cu_credential,
+    file_search=FileSearchConfig.from_foundry(
+        client.client,  # reuse the LLM client's internal AsyncAzureOpenAI for file uploads
+        vector_store_id=_vector_store.id,
+        file_search_tool=_file_search_tool,
+    ),
+)
+
+agent = Agent(
+    client=client,
+    name="FileSearchDocAgent",
+    instructions=(
+        "You are a helpful document analysis assistant with RAG capabilities. "
+        "When a user uploads files, they are automatically analyzed using Azure Content Understanding "
+        "and indexed in a vector store for efficient retrieval. "
+        "Analysis takes time (seconds for documents, longer for audio/video) — if a document "
+        "is still pending, let the user know and suggest they ask again shortly. "
+        "You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
+        "Multiple files can be uploaded and queried in the same conversation. "
+        "When answering, cite specific content from the documents."
+    ),
+    context_providers=[cu],
+)
@@ -0,0 +1,34 @@
+# DevUI Foundry File Search Agent
+
+Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + Foundry file_search RAG.
+
+This is the **Foundry** variant. For the Azure OpenAI Responses API variant, see `devui_azure_openai_file_search_agent`.
+
+## How It Works
+
+1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
+2. **CU analyzes** the file — auto-selects the right analyzer per media type
+3. **Markdown extracted** by CU is uploaded to a Foundry vector store
+4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
+5. **Ask questions** across all uploaded documents with token-efficient RAG
+
+## Setup
+
+1. Set environment variables (or create a `.env` file in `python/`):
+   ```bash
+   FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
+   FOUNDRY_MODEL=gpt-4.1
+   AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
+   ```
+
+2. Log in with Azure CLI:
+   ```bash
+   az login
+   ```
+
+3. Run with DevUI:
+   ```bash
+   devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
+   ```
+
+4. Open the DevUI URL in your browser and start uploading files.
@@ -0,0 +1 @@
+# Copyright (c) Microsoft. All rights reserved.
@@ -0,0 +1,109 @@
+# Copyright (c) Microsoft. All rights reserved.
+"""DevUI Multi-Modal Agent — CU extraction + file_search RAG via Azure AI Foundry.
+
+This agent combines Azure Content Understanding with Foundry's file_search
+for token-efficient RAG over large or multi-modal documents.
+
+Upload flow:
+  1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
+  2. Extracted markdown is uploaded to a Foundry vector store
+  3. file_search tool is registered so the LLM retrieves top-k chunks
+  4. Uploaded files are cleaned up on server shutdown
+
+This sample uses ``FoundryChatClient`` and ``FoundryFileSearchBackend``.
+For the OpenAI Responses API variant, see ``devui_azure_openai_file_search_agent``.
+
+Analyzer auto-detection:
+  When no analyzer_id is specified, the provider auto-selects the
+  appropriate CU analyzer based on media type:
+    - Documents/images → prebuilt-documentSearch
+    - Audio            → prebuilt-audioSearch
+    - Video            → prebuilt-videoSearch
+
+Required environment variables:
+  FOUNDRY_PROJECT_ENDPOINT                 — Azure AI Foundry project endpoint
+  FOUNDRY_MODEL                            — Model deployment name (e.g. gpt-4.1)
+  AZURE_CONTENTUNDERSTANDING_ENDPOINT      — CU endpoint URL
+
+Run with DevUI:
+  devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
+"""
+
+import os
+
+from agent_framework import Agent
+from agent_framework.foundry import (
+    ContentUnderstandingContextProvider,
+    FileSearchConfig,
+    FoundryChatClient,
+)
+from azure.core.credentials import AzureKeyCredential
+from azure.identity import AzureCliCredential
+from dotenv import load_dotenv
+from openai import AzureOpenAI
+
+load_dotenv()
+
+# --- Auth ---
+# AzureCliCredential for Foundry. CU API key optional if on a different resource.
+_credential = AzureCliCredential()
+_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
+_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
+
+# --- Foundry LLM client ---
+client = FoundryChatClient(
+    project_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
+    model=os.environ.get("FOUNDRY_MODEL", ""),
+    credential=_credential,
+)
+
+# --- Create vector store (sync client to avoid event loop conflicts in DevUI) ---
+_token = _credential.get_token("https://ai.azure.com/.default").token
+_sync_openai = AzureOpenAI(
+    azure_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
+    azure_ad_token=_token,
+    api_version="2025-04-01-preview",
+)
+_vector_store = _sync_openai.vector_stores.create(
+    name="devui_cu_foundry_file_search",
+    expires_after={"anchor": "last_active_at", "days": 1},
+)
+_sync_openai.close()
+
+_file_search_tool = client.get_file_search_tool(
+    vector_store_ids=[_vector_store.id],
+    max_num_results=3,  # limit chunks to reduce input token usage
+)
+
+# --- CU context provider with file_search ---
+# No analyzer_id → auto-selects per media type (documents, audio, video)
+cu = ContentUnderstandingContextProvider(
+    endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
+    credential=_cu_credential,
+    # max_wait is the combined budget for CU analysis + vector store upload.
+    # For file_search mode, 10s gives enough time for small documents to be
+    # analyzed and indexed in one turn.  Larger files (audio, video) will
+    # be deferred to background and resolved on the next turn.
+    max_wait=10.0,
+    file_search=FileSearchConfig.from_foundry(
+        client.client,
+        vector_store_id=_vector_store.id,
+        file_search_tool=_file_search_tool,
+    ),
+)
+
+agent = Agent(
+    client=client,
+    name="FoundryFileSearchDocAgent",
+    instructions=(
+        "You are a helpful document analysis assistant with RAG capabilities. "
+        "When a user uploads files, they are automatically analyzed using Azure Content Understanding "
+        "and indexed in a vector store for efficient retrieval. "
+        "Analysis takes time (seconds for documents, longer for audio/video) — if a document "
+        "is still pending, let the user know and suggest they ask again shortly. "
+        "You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
+        "Multiple files can be uploaded and queried in the same conversation. "
+        "When answering, cite specific content from the documents."
+    ),
+    context_providers=[cu],
+)
				`@@ -0,0 +1 @@`
				`# Copyright (c) Microsoft. All rights reserved.`