chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "agent-framework-azure-contentunderstanding",
|
||||
# "agent-framework-foundry",
|
||||
# "azure-identity",
|
||||
# ]
|
||||
# ///
|
||||
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/01_document_qa.py
|
||||
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import Agent, Content, Message
|
||||
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Document Q&A — PDF upload with CU-powered extraction
|
||||
|
||||
This sample demonstrates the simplest CU integration: upload a PDF and
|
||||
ask questions about it. Azure Content Understanding extracts structured
|
||||
markdown with table preservation — superior to LLM-only vision for
|
||||
scanned PDFs, handwritten content, and complex layouts.
|
||||
|
||||
Environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
"""
|
||||
|
||||
# Path to a sample PDF — uses the shared sample asset if available,
|
||||
# otherwise falls back to a public URL
|
||||
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
credential = AzureCliCredential()
|
||||
|
||||
# Set up Azure Content Understanding context provider
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=credential,
|
||||
analyzer_id="prebuilt-documentSearch", # RAG-optimized document analyzer
|
||||
max_wait=None, # wait until CU analysis finishes (no background deferral)
|
||||
)
|
||||
|
||||
# Set up the LLM client
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
# Create agent with CU context provider.
|
||||
# The provider extracts document content via CU and injects it into the
|
||||
# LLM context so the agent can answer questions about the document.
|
||||
async with cu:
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="DocumentQA",
|
||||
instructions=(
|
||||
"You are a helpful document analyst. Use the analyzed document "
|
||||
"content and extracted fields to answer questions precisely."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
|
||||
# --- Turn 1: Upload PDF and ask a question ---
|
||||
# 4. Upload PDF and ask questions
|
||||
# The CU provider extracts markdown + fields from the PDF and injects
|
||||
# the full content into context so the agent can answer precisely.
|
||||
print("--- Upload PDF and ask questions ---")
|
||||
|
||||
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
|
||||
|
||||
response = await agent.run(
|
||||
Message(
|
||||
role="user",
|
||||
contents=[
|
||||
Content.from_text(
|
||||
"What is this document about? Who is the vendor, and what is the total amount due?"
|
||||
),
|
||||
Content.from_data(
|
||||
pdf_bytes,
|
||||
"application/pdf",
|
||||
# Always provide filename — used as the document key
|
||||
additional_properties={"filename": SAMPLE_PDF_PATH.name},
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
usage = response.usage_details or {}
|
||||
print(f"Agent: {response}")
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
"""
|
||||
Sample output:
|
||||
|
||||
--- Upload PDF and ask questions ---
|
||||
Agent: This document is an **invoice** for services and fees billed to
|
||||
**MICROSOFT CORPORATION** (Invoice **INV-100**), including line items
|
||||
(e.g., Consulting Services, Document Fee, Printing Fee) and a billing summary.
|
||||
- **Vendor:** **CONTOSO LTD.**
|
||||
- **Total amount due:** **$610.00**
|
||||
[Input tokens: 988]
|
||||
"""
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "agent-framework-azure-contentunderstanding",
|
||||
# "agent-framework-foundry",
|
||||
# "azure-identity",
|
||||
# ]
|
||||
# ///
|
||||
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/02_multi_turn_session.py
|
||||
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import Agent, AgentSession, Content, Message
|
||||
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Multi-Turn Session — Cached results across turns
|
||||
|
||||
This sample demonstrates multi-turn document Q&A using an AgentSession.
|
||||
The session persists CU analysis results and conversation history across
|
||||
turns so the agent can answer follow-up questions about previously
|
||||
uploaded documents without re-analyzing them.
|
||||
|
||||
Key concepts:
|
||||
- AgentSession keeps CU state and conversation history across agent.run() calls
|
||||
- Turn 1: CU analyzes the PDF and injects full content into context
|
||||
- Turn 2: Unrelated question — agent answers from general knowledge
|
||||
- Turn 3: Detailed question — agent uses document content from conversation
|
||||
history (injected in Turn 1) to answer precisely
|
||||
|
||||
Environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
"""
|
||||
|
||||
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# 1. Set up credentials and CU context provider
|
||||
credential = AzureCliCredential()
|
||||
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=credential,
|
||||
analyzer_id="prebuilt-documentSearch",
|
||||
max_wait=None, # wait until CU analysis finishes (no background deferral)
|
||||
)
|
||||
|
||||
# 2. Set up the LLM client
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
# 3. Create agent and persistent session
|
||||
async with cu:
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="DocumentQA",
|
||||
instructions=(
|
||||
"You are a helpful document analyst. Use the analyzed document "
|
||||
"content and extracted fields to answer questions precisely."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
|
||||
# Create a persistent session — this keeps CU state across turns
|
||||
session = AgentSession()
|
||||
|
||||
# 4. Turn 1: Upload PDF
|
||||
# CU analyzes the PDF and injects full content into context.
|
||||
print("--- Turn 1: Upload PDF ---")
|
||||
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
|
||||
response = await agent.run(
|
||||
Message(
|
||||
role="user",
|
||||
contents=[
|
||||
Content.from_text("What is this document about?"),
|
||||
Content.from_data(
|
||||
pdf_bytes,
|
||||
"application/pdf",
|
||||
additional_properties={"filename": SAMPLE_PDF_PATH.name},
|
||||
),
|
||||
],
|
||||
),
|
||||
session=session, # <-- persist state across turns
|
||||
)
|
||||
usage = response.usage_details or {}
|
||||
print(f"Agent: {response}")
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
|
||||
|
||||
# 5. Turn 2: Unrelated question
|
||||
# No document needed — agent answers from general knowledge.
|
||||
print("--- Turn 2: Unrelated question ---")
|
||||
response = await agent.run("What is the capital of France?", session=session)
|
||||
usage = response.usage_details or {}
|
||||
print(f"Agent: {response}")
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
|
||||
|
||||
# 6. Turn 3: Detailed follow-up
|
||||
# The agent answers from the full document content that was injected
|
||||
# into conversation history in Turn 1. No re-analysis or tool call needed.
|
||||
print("--- Turn 3: Detailed follow-up ---")
|
||||
response = await agent.run(
|
||||
"What is the shipping address on the invoice?",
|
||||
session=session,
|
||||
)
|
||||
usage = response.usage_details or {}
|
||||
print(f"Agent: {response}")
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
"""
|
||||
Sample output:
|
||||
|
||||
--- Turn 1: Upload PDF ---
|
||||
Agent: This document is an **invoice** from **CONTOSO LTD.** to **MICROSOFT
|
||||
CORPORATION**. Amount Due: $610.00. Invoice INV-100, dated 11/15/2019.
|
||||
[Input tokens: 975]
|
||||
|
||||
--- Turn 2: Unrelated question ---
|
||||
Agent: Paris.
|
||||
[Input tokens: 1134]
|
||||
|
||||
--- Turn 3: Detailed follow-up ---
|
||||
Agent: Shipping address (SHIP TO): Microsoft Delivery, 123 Ship St,
|
||||
Redmond WA, 98052.
|
||||
[Input tokens: 1155]
|
||||
"""
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "agent-framework-azure-contentunderstanding",
|
||||
# "agent-framework-foundry",
|
||||
# "azure-identity",
|
||||
# ]
|
||||
# ///
|
||||
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/03_multimodal_chat.py
|
||||
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import Agent, AgentSession, Content, Message
|
||||
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Multi-Modal Chat — PDF, audio, and video in a single turn
|
||||
|
||||
This sample demonstrates CU's multi-modal capability: upload a PDF invoice,
|
||||
an audio call recording, and a video file all at once. The provider analyzes
|
||||
all three in parallel using the right CU analyzer for each media type.
|
||||
|
||||
The provider auto-detects the media type and selects the right CU analyzer:
|
||||
- PDF/images → prebuilt-documentSearch
|
||||
- Audio → prebuilt-audioSearch
|
||||
- Video → prebuilt-videoSearch
|
||||
|
||||
Environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
"""
|
||||
|
||||
# Local PDF from package assets
|
||||
SAMPLE_PDF = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
|
||||
|
||||
# Public audio/video from Azure CU samples repo (raw GitHub URLs)
|
||||
_CU_ASSETS = "https://raw.githubusercontent.com/Azure-Samples/azure-ai-content-understanding-assets/main"
|
||||
AUDIO_URL = f"{_CU_ASSETS}/audio/callCenterRecording.mp3"
|
||||
VIDEO_URL = f"{_CU_ASSETS}/videos/sdk_samples/FlightSimulator.mp4"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# 1. Set up credentials and CU context provider
|
||||
credential = AzureCliCredential()
|
||||
|
||||
# No analyzer_id specified — the provider auto-detects from media type:
|
||||
# PDF/images → prebuilt-documentSearch
|
||||
# Audio → prebuilt-audioSearch
|
||||
# Video → prebuilt-videoSearch
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=credential,
|
||||
max_wait=None, # wait until each analysis finishes
|
||||
)
|
||||
|
||||
# 2. Set up the LLM client
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
# 3. Create agent and session
|
||||
async with cu:
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="MultiModalAgent",
|
||||
instructions=(
|
||||
"You are a helpful assistant that can analyze documents, audio, "
|
||||
"and video files. Answer questions using the extracted content."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
|
||||
session = AgentSession()
|
||||
|
||||
# --- Turn 1: Upload all 3 modalities at once ---
|
||||
# The provider analyzes all files in parallel using the appropriate
|
||||
# CU analyzer for each media type. All results are injected into
|
||||
# the same context so the agent can answer about all of them.
|
||||
turn1_prompt = (
|
||||
"I'm uploading three files: an invoice PDF, a call center "
|
||||
"audio recording, and a flight simulator video. "
|
||||
"Give a brief summary of each file."
|
||||
)
|
||||
print("--- Turn 1: Upload PDF + audio + video (parallel analysis) ---")
|
||||
print(" (CU analysis may take a few minutes for these audio/video files...)")
|
||||
print(f"User: {turn1_prompt}")
|
||||
t0 = time.perf_counter()
|
||||
response = await agent.run(
|
||||
Message(
|
||||
role="user",
|
||||
contents=[
|
||||
Content.from_text(turn1_prompt),
|
||||
Content.from_data(
|
||||
SAMPLE_PDF.read_bytes(),
|
||||
"application/pdf",
|
||||
additional_properties={"filename": "invoice.pdf"},
|
||||
),
|
||||
Content.from_uri(
|
||||
AUDIO_URL,
|
||||
media_type="audio/mp3",
|
||||
additional_properties={"filename": "callCenterRecording.mp3"},
|
||||
),
|
||||
Content.from_uri(
|
||||
VIDEO_URL,
|
||||
media_type="video/mp4",
|
||||
additional_properties={"filename": "FlightSimulator.mp4"},
|
||||
),
|
||||
],
|
||||
),
|
||||
session=session,
|
||||
)
|
||||
elapsed = time.perf_counter() - t0
|
||||
usage = response.usage_details or {}
|
||||
print(f" [Analyzed in {elapsed:.1f}s | Input tokens: {usage.get('input_token_count', 'N/A')}]")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# --- Turn 2: Detail question about the PDF ---
|
||||
turn2_prompt = "What are the line items and their amounts on the invoice?"
|
||||
print("--- Turn 2: PDF detail ---")
|
||||
print(f"User: {turn2_prompt}")
|
||||
response = await agent.run(turn2_prompt, session=session)
|
||||
usage = response.usage_details or {}
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# --- Turn 3: Detail question about the audio ---
|
||||
turn3_prompt = "What was the customer's issue in the call recording?"
|
||||
print("--- Turn 3: Audio detail ---")
|
||||
print(f"User: {turn3_prompt}")
|
||||
response = await agent.run(turn3_prompt, session=session)
|
||||
usage = response.usage_details or {}
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# --- Turn 4: Detail question about the video ---
|
||||
turn4_prompt = "What key scenes or actions are shown in the flight simulator video?"
|
||||
print("--- Turn 4: Video detail ---")
|
||||
print(f"User: {turn4_prompt}")
|
||||
response = await agent.run(turn4_prompt, session=session)
|
||||
usage = response.usage_details or {}
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# --- Turn 5: Cross-document question ---
|
||||
turn5_prompt = (
|
||||
"Across all three files, which one contains financial data, "
|
||||
"which one involves a customer interaction, and which one is "
|
||||
"a visual demonstration?"
|
||||
)
|
||||
print("--- Turn 5: Cross-document question ---")
|
||||
print(f"User: {turn5_prompt}")
|
||||
response = await agent.run(turn5_prompt, session=session)
|
||||
usage = response.usage_details or {}
|
||||
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
"""
|
||||
Sample output:
|
||||
|
||||
--- Turn 1: Upload PDF + audio + video (parallel analysis) ---
|
||||
User: I'm uploading three files...
|
||||
(CU analysis may take 1-2 minutes for audio/video files...)
|
||||
[Analyzed in ~94s | Input tokens: ~2939]
|
||||
Agent: ### invoice.pdf: An invoice from CONTOSO LTD. to MICROSOFT CORPORATION...
|
||||
### callCenterRecording.mp3: A customer service call about point balance...
|
||||
### FlightSimulator.mp4: A clip discussing neural text-to-speech...
|
||||
|
||||
--- Turn 2-5: Detail and cross-document questions ---
|
||||
(Agent answers from conversation history without re-analysis)
|
||||
"""
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "agent-framework-azure-contentunderstanding",
|
||||
# "agent-framework-foundry",
|
||||
# "azure-identity",
|
||||
# "pydantic",
|
||||
# ]
|
||||
# ///
|
||||
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/04_invoice_processing.py
|
||||
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import Agent, AgentSession, Content, Message
|
||||
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Invoice Processing — Structured output with prebuilt-invoice analyzer
|
||||
|
||||
This sample demonstrates CU's structured field extraction combined with
|
||||
LLM structured output (Pydantic model). The prebuilt-invoice analyzer extracts
|
||||
typed fields (VendorName, InvoiceTotal, DueDate, LineItems, etc.) with
|
||||
confidence scores. We use output_sections=["fields"] only (no markdown needed)
|
||||
since we want the LLM to produce a structured JSON response from the extracted
|
||||
fields, not summarize document text.
|
||||
|
||||
Environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
"""
|
||||
|
||||
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
|
||||
|
||||
|
||||
# Structured output model — the LLM will return JSON matching this schema
|
||||
# Structured output models — the LLM returns JSON matching this schema.
|
||||
#
|
||||
# Note: the prebuilt-invoice analyzer extracts an extensive set of fields
|
||||
# (VendorName, BillingAddress, ShippingAddress, TaxDetails, PONumber, etc.).
|
||||
# This sample defines a simplified schema to extract only the fields of
|
||||
# interest to the caller. The LLM maps the full CU field output to this
|
||||
# subset automatically.
|
||||
# Learn more about prebuilt analyzers: https://learn.microsoft.com/azure/ai-services/content-understanding/concepts/prebuilt-analyzers
|
||||
|
||||
|
||||
class LineItem(BaseModel):
|
||||
description: str
|
||||
quantity: float | None = None
|
||||
unit_price: float | None = None
|
||||
amount: float | None = None
|
||||
|
||||
|
||||
class LowConfidenceField(BaseModel):
|
||||
field_name: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class InvoiceResult(BaseModel):
|
||||
vendor_name: str
|
||||
total_amount: float | None = None
|
||||
currency: str = "USD"
|
||||
due_date: str | None = None
|
||||
line_items: list[LineItem] = Field(default_factory=list)
|
||||
low_confidence_fields: list[LowConfidenceField] = Field(
|
||||
default_factory=list,
|
||||
description="Fields with confidence < 0.8, including their confidence score",
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# 1. Set up credentials and CU context provider
|
||||
credential = AzureCliCredential()
|
||||
|
||||
# Default analyzer is prebuilt-documentSearch (RAG-optimized).
|
||||
# Per-file override via additional_properties["analyzer_id"] lets us
|
||||
# use prebuilt-invoice for structured field extraction on specific files.
|
||||
#
|
||||
# Only request "fields" (not "markdown") — we want the extracted typed
|
||||
# fields for structured output, not the raw document text.
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=credential,
|
||||
analyzer_id="prebuilt-documentSearch", # default for all files
|
||||
max_wait=None, # wait until CU analysis finishes
|
||||
output_sections=["fields"], # fields only — structured output doesn't need markdown
|
||||
)
|
||||
|
||||
# 2. Set up the LLM client
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
# 3. Create agent and session
|
||||
async with cu:
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="InvoiceProcessor",
|
||||
instructions=(
|
||||
"You are an invoice processing assistant. Extract invoice data from "
|
||||
"the provided CU fields (JSON with confidence scores). Return structured "
|
||||
"output matching the requested schema. Flag fields with confidence < 0.8 "
|
||||
"in the low_confidence_fields list."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
|
||||
session = AgentSession()
|
||||
|
||||
# 4. Upload an invoice PDF — uses structured output (Pydantic model)
|
||||
print("--- Upload Invoice (Structured Output) ---")
|
||||
|
||||
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
|
||||
|
||||
response = await agent.run(
|
||||
Message(
|
||||
role="user",
|
||||
contents=[
|
||||
Content.from_text(
|
||||
"Process this invoice. Extract the vendor name, total amount, due date, and all line items."
|
||||
),
|
||||
Content.from_data(
|
||||
pdf_bytes,
|
||||
"application/pdf",
|
||||
# Per-file analyzer override: use prebuilt-invoice for
|
||||
# structured field extraction (VendorName, InvoiceTotal, etc.)
|
||||
# instead of the provider default (prebuilt-documentSearch).
|
||||
additional_properties={
|
||||
"filename": SAMPLE_PDF_PATH.name,
|
||||
"analyzer_id": "prebuilt-invoice",
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
session=session,
|
||||
options={"response_format": InvoiceResult},
|
||||
)
|
||||
|
||||
# Parse the structured output from JSON text
|
||||
try:
|
||||
invoice = InvoiceResult.model_validate_json(response.text)
|
||||
print(f"Vendor: {invoice.vendor_name}")
|
||||
print(f"Total: {invoice.currency} {invoice.total_amount}")
|
||||
print(f"Due date: {invoice.due_date}")
|
||||
print(f"Line items ({len(invoice.line_items)}):")
|
||||
for item in invoice.line_items:
|
||||
print(f" - {item.description}: {item.amount}")
|
||||
if invoice.low_confidence_fields:
|
||||
print("⚠ Low confidence fields:")
|
||||
for f in invoice.low_confidence_fields:
|
||||
print(f" - {f.field_name}: {f.confidence:.3f}")
|
||||
except Exception:
|
||||
print(f"Agent (raw): {response.text}\n")
|
||||
|
||||
# 5. Follow-up: free-text question about the invoice
|
||||
print("\n--- Follow-up (Free Text) ---")
|
||||
response = await agent.run(
|
||||
"What is the payment term? Are there any fields with low confidence?",
|
||||
session=session,
|
||||
)
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
"""
|
||||
Sample output:
|
||||
|
||||
--- Upload Invoice (Structured Output) ---
|
||||
Vendor: CONTOSO LTD.
|
||||
Total: USD 110.0
|
||||
Due date: 2019-12-15
|
||||
Line items (3):
|
||||
- Consulting Services: 60.0
|
||||
- Document Fee: 30.0
|
||||
- Printing Fee: 10.0
|
||||
⚠ Low confidence: VendorName, CustomerName
|
||||
|
||||
--- Follow-up (Free Text) ---
|
||||
Agent: The payment terms are not explicitly stated on the invoice...
|
||||
"""
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "agent-framework-azure-contentunderstanding",
|
||||
# "agent-framework-foundry",
|
||||
# "azure-identity",
|
||||
# ]
|
||||
# ///
|
||||
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/05_large_doc_file_search.py
|
||||
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import Agent, AgentSession, Content, Message
|
||||
from agent_framework.foundry import (
|
||||
ContentUnderstandingContextProvider,
|
||||
FileSearchConfig,
|
||||
FoundryChatClient,
|
||||
)
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Large Document + file_search RAG — CU extraction + OpenAI vector store
|
||||
|
||||
For large documents (100+ pages) or long audio/video, injecting the full
|
||||
CU-extracted content into the LLM context is impractical. This sample shows
|
||||
how to use the built-in file_search integration: CU extracts markdown and
|
||||
automatically uploads it to an OpenAI vector store for token-efficient RAG.
|
||||
|
||||
When ``FileSearchConfig`` is provided, the provider:
|
||||
1. Extracts markdown via CU (handles scanned PDFs, audio, video)
|
||||
2. Uploads the extracted markdown to a vector store
|
||||
3. Registers a ``file_search`` tool on the agent context
|
||||
4. Cleans up the vector store on close
|
||||
|
||||
Architecture:
|
||||
Large PDF -> CU extracts markdown -> auto-upload to vector store -> file_search
|
||||
Follow-up -> file_search retrieves top-k chunks -> LLM answers
|
||||
|
||||
NOTE: Requires an async OpenAI client for vector store operations.
|
||||
|
||||
This sample uses a single small invoice PDF for simplicity. In practice,
|
||||
you can upload multiple files in the same session (each is indexed
|
||||
separately in the vector store), and this pattern is most valuable for
|
||||
large documents (up to 300 pages), long audio recordings, or video files
|
||||
where full-context injection would exceed the LLM's context window.
|
||||
CU supports PDFs up to 300 pages / 200 MB, and audio files up to 300 MB
|
||||
— see the full service limits:
|
||||
https://learn.microsoft.com/azure/ai-services/content-understanding/service-limits#input-file-limits
|
||||
|
||||
Environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
"""
|
||||
|
||||
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# 1. Set up credentials and LLM client
|
||||
credential = AzureCliCredential()
|
||||
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
# 2. Get the async OpenAI client from FoundryChatClient for vector store operations
|
||||
openai_client = client.client
|
||||
|
||||
# 3. Create vector store and file_search tool
|
||||
vector_store = await openai_client.vector_stores.create(
|
||||
name="cu_large_doc_demo",
|
||||
expires_after={"anchor": "last_active_at", "days": 1},
|
||||
)
|
||||
file_search_tool = client.get_file_search_tool(vector_store_ids=[vector_store.id])
|
||||
|
||||
# 4. Configure CU provider with file_search integration
|
||||
# When file_search is set, CU-extracted markdown is automatically uploaded
|
||||
# to the vector store and the file_search tool is registered on the context.
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=credential,
|
||||
analyzer_id="prebuilt-documentSearch",
|
||||
max_wait=None, # wait until CU analysis + vector store upload finishes
|
||||
file_search=FileSearchConfig.from_foundry(
|
||||
openai_client,
|
||||
vector_store_id=vector_store.id,
|
||||
file_search_tool=file_search_tool,
|
||||
),
|
||||
)
|
||||
|
||||
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
|
||||
|
||||
# The provider handles everything: CU extraction + vector store upload + file_search tool
|
||||
async with cu:
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="LargeDocAgent",
|
||||
instructions=(
|
||||
"You are a document analyst. Use the file_search tool to find "
|
||||
"relevant sections from the document and answer precisely. "
|
||||
"Cite specific sections when answering."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
|
||||
session = AgentSession()
|
||||
|
||||
# Turn 1: Upload — CU extracts and uploads to vector store automatically
|
||||
print("--- Turn 1: Upload document ---")
|
||||
response = await agent.run(
|
||||
Message(
|
||||
role="user",
|
||||
contents=[
|
||||
Content.from_text("What are the key points in this document?"),
|
||||
Content.from_data(
|
||||
pdf_bytes,
|
||||
"application/pdf",
|
||||
additional_properties={"filename": SAMPLE_PDF_PATH.name},
|
||||
),
|
||||
],
|
||||
),
|
||||
session=session,
|
||||
)
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# Turn 2: Follow-up — file_search retrieves relevant chunks (token efficient)
|
||||
print("--- Turn 2: Follow-up (RAG) ---")
|
||||
response = await agent.run(
|
||||
"What numbers or financial metrics are mentioned?",
|
||||
session=session,
|
||||
)
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# Explicitly delete the vector store created for this sample
|
||||
await openai_client.vector_stores.delete(vector_store.id)
|
||||
print("Done. Vector store deleted.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
"""
|
||||
Sample output:
|
||||
|
||||
--- Turn 1: Upload document ---
|
||||
Agent: An invoice from Contoso Ltd. to Microsoft Corporation (INV-100).
|
||||
Line items: Consulting Services $60, Document Fee $30, Printing Fee $10.
|
||||
Subtotal $100, Sales tax $10, Total $110, Previous balance $500, Amount due $610.
|
||||
|
||||
--- Turn 2: Follow-up (RAG) ---
|
||||
Agent: Subtotal $100.00, Sales tax $10.00, Total $110.00,
|
||||
Previous unpaid balance $500.00, Amount due $610.00.
|
||||
Line items: 2 hours @ $30 = $60, 3 @ $10 = $30, 10 pages @ $1 = $10.
|
||||
|
||||
Done. Vector store cleaned up automatically.
|
||||
"""
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
# DevUI Multi-Modal Agent
|
||||
|
||||
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding.
|
||||
|
||||
## Setup
|
||||
|
||||
1. Set environment variables (or create a `.env` file in `python/`):
|
||||
```bash
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://your-project.api.azureml.ms
|
||||
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
|
||||
```
|
||||
|
||||
2. Log in with Azure CLI:
|
||||
```bash
|
||||
az login
|
||||
```
|
||||
|
||||
3. Run with DevUI:
|
||||
```bash
|
||||
uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
|
||||
```
|
||||
|
||||
4. Open the DevUI URL in your browser and start uploading files.
|
||||
|
||||
## What You Can Do
|
||||
|
||||
- **Upload PDFs** — including scanned/image-based PDFs that LLM vision struggles with
|
||||
- **Upload images** — handwritten notes, infographics, charts
|
||||
- **Upload audio** — meeting recordings, call center calls (transcription with speaker ID)
|
||||
- **Upload video** — product demos, training videos (frame extraction + transcription)
|
||||
- **Ask questions** across all uploaded documents
|
||||
- **Check status** — "which documents are ready?" uses the auto-registered `list_documents()` tool
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""DevUI Multi-Modal Agent with Azure Content Understanding."""
|
||||
|
||||
from .agent import agent
|
||||
|
||||
__all__ = ["agent"]
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""DevUI Multi-Modal Agent — file upload + CU-powered analysis.
|
||||
|
||||
This agent uses Azure Content Understanding to analyze uploaded files
|
||||
(PDFs, scanned documents, handwritten images, audio recordings, video)
|
||||
and answer questions about them through the DevUI web interface.
|
||||
|
||||
Unlike the standard azure_responses_agent which sends files directly to the LLM,
|
||||
this agent uses CU for structured extraction — superior for scanned PDFs,
|
||||
handwritten content, audio transcription, and video analysis.
|
||||
|
||||
Required environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
|
||||
Run with DevUI:
|
||||
uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from agent_framework import Agent
|
||||
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# --- Auth ---
|
||||
_credential = AzureCliCredential()
|
||||
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
|
||||
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
|
||||
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=_cu_credential,
|
||||
# max_wait controls how long before_run() waits for CU analysis before
|
||||
# deferring to background. For interactive DevUI use, a short timeout
|
||||
# (e.g. 5s) keeps the chat responsive — the agent tells the user the
|
||||
# file is still being analyzed and resolves it on the next turn.
|
||||
# Use max_wait=None to always wait for analysis to complete.
|
||||
max_wait=5.0,
|
||||
)
|
||||
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=_credential,
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="MultiModalDocAgent",
|
||||
instructions=(
|
||||
"You are a helpful document analysis assistant. "
|
||||
"When a user uploads files, they are automatically analyzed using Azure Content Understanding. "
|
||||
"Use list_documents() to check which documents are ready, pending, or failed "
|
||||
"and to see which files are available for answering questions. "
|
||||
"Tell the user if any documents are still being analyzed. "
|
||||
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
|
||||
"When answering, cite specific content from the documents."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# DevUI File Search Agent
|
||||
|
||||
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + OpenAI file_search RAG.
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
|
||||
2. **CU analyzes** the file — auto-selects the right analyzer per media type
|
||||
3. **Markdown extracted** by CU is uploaded to an OpenAI vector store
|
||||
4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
|
||||
5. **Ask questions** across all uploaded documents with token-efficient RAG
|
||||
|
||||
## Setup
|
||||
|
||||
1. Set environment variables (or create a `.env` file in `python/`):
|
||||
```bash
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
|
||||
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
|
||||
```
|
||||
|
||||
2. Log in with Azure CLI:
|
||||
```bash
|
||||
az login
|
||||
```
|
||||
|
||||
3. Run with DevUI:
|
||||
```bash
|
||||
devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
|
||||
```
|
||||
|
||||
4. Open the DevUI URL in your browser and start uploading files.
|
||||
|
||||
## Supported File Types
|
||||
|
||||
| Type | Formats | CU Analyzer (auto-detected) |
|
||||
|------|---------|----------------------------|
|
||||
| Documents | PDF, DOCX, XLSX, PPTX, HTML, TXT, Markdown | `prebuilt-documentSearch` |
|
||||
| Images | JPEG, PNG, TIFF, BMP | `prebuilt-documentSearch` |
|
||||
| Audio | WAV, MP3, FLAC, OGG, M4A | `prebuilt-audioSearch` |
|
||||
| Video | MP4, MOV, AVI, WebM | `prebuilt-videoSearch` |
|
||||
|
||||
## vs. devui_multimodal_agent
|
||||
|
||||
| Feature | multimodal_agent | file_search_agent |
|
||||
|---------|-----------------|-------------------|
|
||||
| CU extraction | ✅ Full content injected | ✅ Content indexed in vector store |
|
||||
| RAG | ❌ | ✅ file_search retrieves top-k chunks |
|
||||
| Large docs (100+ pages) | ⚠️ May exceed context window | ✅ Token-efficient |
|
||||
| Multiple large files | ⚠️ Context overflow risk | ✅ All indexed, searchable |
|
||||
| Best for | Small docs, quick inspection | Large docs, multi-file Q&A |
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""DevUI Multi-Modal Agent with CU + file_search RAG."""
|
||||
|
||||
from .agent import agent
|
||||
|
||||
__all__ = ["agent"]
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""DevUI Multi-Modal Agent — CU extraction + file_search RAG.
|
||||
|
||||
This agent combines Azure Content Understanding with OpenAI file_search
|
||||
for token-efficient RAG over large or multi-modal documents.
|
||||
|
||||
Upload flow:
|
||||
1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
|
||||
2. Extracted markdown is auto-uploaded to an OpenAI vector store
|
||||
3. file_search tool is registered so the LLM retrieves top-k chunks
|
||||
4. Vector store is configured to auto-expire after inactivity
|
||||
|
||||
This is ideal for large documents (100+ pages), long audio recordings,
|
||||
or multiple files in the same conversation where full-context injection
|
||||
would exceed the LLM's context window.
|
||||
|
||||
Analyzer auto-detection:
|
||||
When no analyzer_id is specified, the provider auto-selects the
|
||||
appropriate CU analyzer based on media type:
|
||||
- Documents/images → prebuilt-documentSearch
|
||||
- Audio → prebuilt-audioSearch
|
||||
- Video → prebuilt-videoSearch
|
||||
|
||||
Required environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
|
||||
Run with DevUI:
|
||||
devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from agent_framework import Agent
|
||||
from agent_framework.foundry import (
|
||||
ContentUnderstandingContextProvider,
|
||||
FileSearchConfig,
|
||||
FoundryChatClient,
|
||||
)
|
||||
from azure.ai.projects import AIProjectClient
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# --- Auth ---
|
||||
_credential = AzureCliCredential()
|
||||
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
|
||||
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
|
||||
|
||||
_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
|
||||
|
||||
# --- LLM client + sync vector store setup ---
|
||||
# DevUI loads agent modules synchronously at startup while an event loop is already
|
||||
# running, so we cannot use async APIs here. A sync AIProjectClient is used for
|
||||
# one-time vector store creation; runtime file uploads use client.client (async).
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=_endpoint,
|
||||
model=os.environ["FOUNDRY_MODEL"],
|
||||
credential=_credential,
|
||||
)
|
||||
|
||||
_sync_project = AIProjectClient(endpoint=_endpoint, credential=_credential) # type: ignore[arg-type]
|
||||
_sync_openai = _sync_project.get_openai_client()
|
||||
_vector_store = _sync_openai.vector_stores.create(
|
||||
name="devui_cu_file_search",
|
||||
expires_after={"anchor": "last_active_at", "days": 1},
|
||||
)
|
||||
_sync_openai.close()
|
||||
|
||||
_file_search_tool = client.get_file_search_tool(
|
||||
vector_store_ids=[_vector_store.id],
|
||||
max_num_results=3, # limit chunks to reduce input token usage
|
||||
)
|
||||
|
||||
# --- CU context provider with file_search ---
|
||||
# client.client is the async OpenAI client used for runtime file uploads.
|
||||
# No analyzer_id → auto-selects per media type (documents, audio, video)
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=_cu_credential,
|
||||
file_search=FileSearchConfig.from_foundry(
|
||||
client.client, # reuse the LLM client's internal AsyncAzureOpenAI for file uploads
|
||||
vector_store_id=_vector_store.id,
|
||||
file_search_tool=_file_search_tool,
|
||||
),
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="FileSearchDocAgent",
|
||||
instructions=(
|
||||
"You are a helpful document analysis assistant with RAG capabilities. "
|
||||
"When a user uploads files, they are automatically analyzed using Azure Content Understanding "
|
||||
"and indexed in a vector store for efficient retrieval. "
|
||||
"Analysis takes time (seconds for documents, longer for audio/video) — if a document "
|
||||
"is still pending, let the user know and suggest they ask again shortly. "
|
||||
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
|
||||
"Multiple files can be uploaded and queried in the same conversation. "
|
||||
"When answering, cite specific content from the documents."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# DevUI Foundry File Search Agent
|
||||
|
||||
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + Foundry file_search RAG.
|
||||
|
||||
This is the **Foundry** variant. For the Azure OpenAI Responses API variant, see `devui_azure_openai_file_search_agent`.
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
|
||||
2. **CU analyzes** the file — auto-selects the right analyzer per media type
|
||||
3. **Markdown extracted** by CU is uploaded to a Foundry vector store
|
||||
4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
|
||||
5. **Ask questions** across all uploaded documents with token-efficient RAG
|
||||
|
||||
## Setup
|
||||
|
||||
1. Set environment variables (or create a `.env` file in `python/`):
|
||||
```bash
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
|
||||
FOUNDRY_MODEL=gpt-4.1
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
|
||||
```
|
||||
|
||||
2. Log in with Azure CLI:
|
||||
```bash
|
||||
az login
|
||||
```
|
||||
|
||||
3. Run with DevUI:
|
||||
```bash
|
||||
devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
|
||||
```
|
||||
|
||||
4. Open the DevUI URL in your browser and start uploading files.
|
||||
+1
@@ -0,0 +1 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""DevUI Multi-Modal Agent — CU extraction + file_search RAG via Azure AI Foundry.
|
||||
|
||||
This agent combines Azure Content Understanding with Foundry's file_search
|
||||
for token-efficient RAG over large or multi-modal documents.
|
||||
|
||||
Upload flow:
|
||||
1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
|
||||
2. Extracted markdown is uploaded to a Foundry vector store
|
||||
3. file_search tool is registered so the LLM retrieves top-k chunks
|
||||
4. Uploaded files are cleaned up on server shutdown
|
||||
|
||||
This sample uses ``FoundryChatClient`` and ``FoundryFileSearchBackend``.
|
||||
For the OpenAI Responses API variant, see ``devui_azure_openai_file_search_agent``.
|
||||
|
||||
Analyzer auto-detection:
|
||||
When no analyzer_id is specified, the provider auto-selects the
|
||||
appropriate CU analyzer based on media type:
|
||||
- Documents/images → prebuilt-documentSearch
|
||||
- Audio → prebuilt-audioSearch
|
||||
- Video → prebuilt-videoSearch
|
||||
|
||||
Required environment variables:
|
||||
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
||||
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
||||
|
||||
Run with DevUI:
|
||||
devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from agent_framework import Agent
|
||||
from agent_framework.foundry import (
|
||||
ContentUnderstandingContextProvider,
|
||||
FileSearchConfig,
|
||||
FoundryChatClient,
|
||||
)
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
from openai import AzureOpenAI
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# --- Auth ---
|
||||
# AzureCliCredential for Foundry. CU API key optional if on a different resource.
|
||||
_credential = AzureCliCredential()
|
||||
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
|
||||
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
|
||||
|
||||
# --- Foundry LLM client ---
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
|
||||
model=os.environ.get("FOUNDRY_MODEL", ""),
|
||||
credential=_credential,
|
||||
)
|
||||
|
||||
# --- Create vector store (sync client to avoid event loop conflicts in DevUI) ---
|
||||
_token = _credential.get_token("https://ai.azure.com/.default").token
|
||||
_sync_openai = AzureOpenAI(
|
||||
azure_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
|
||||
azure_ad_token=_token,
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
_vector_store = _sync_openai.vector_stores.create(
|
||||
name="devui_cu_foundry_file_search",
|
||||
expires_after={"anchor": "last_active_at", "days": 1},
|
||||
)
|
||||
_sync_openai.close()
|
||||
|
||||
_file_search_tool = client.get_file_search_tool(
|
||||
vector_store_ids=[_vector_store.id],
|
||||
max_num_results=3, # limit chunks to reduce input token usage
|
||||
)
|
||||
|
||||
# --- CU context provider with file_search ---
|
||||
# No analyzer_id → auto-selects per media type (documents, audio, video)
|
||||
cu = ContentUnderstandingContextProvider(
|
||||
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
||||
credential=_cu_credential,
|
||||
# max_wait is the combined budget for CU analysis + vector store upload.
|
||||
# For file_search mode, 10s gives enough time for small documents to be
|
||||
# analyzed and indexed in one turn. Larger files (audio, video) will
|
||||
# be deferred to background and resolved on the next turn.
|
||||
max_wait=10.0,
|
||||
file_search=FileSearchConfig.from_foundry(
|
||||
client.client,
|
||||
vector_store_id=_vector_store.id,
|
||||
file_search_tool=_file_search_tool,
|
||||
),
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
client=client,
|
||||
name="FoundryFileSearchDocAgent",
|
||||
instructions=(
|
||||
"You are a helpful document analysis assistant with RAG capabilities. "
|
||||
"When a user uploads files, they are automatically analyzed using Azure Content Understanding "
|
||||
"and indexed in a vector store for efficient retrieval. "
|
||||
"Analysis takes time (seconds for documents, longer for audio/video) — if a document "
|
||||
"is still pending, let the user know and suggest they ask again shortly. "
|
||||
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
|
||||
"Multiple files can be uploaded and queried in the same conversation. "
|
||||
"When answering, cite specific content from the documents."
|
||||
),
|
||||
context_providers=[cu],
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
# Azure Content Understanding Samples
|
||||
|
||||
These samples demonstrate how to use the `agent-framework-azure-contentunderstanding` package to add document, image, audio, and video understanding to your agents.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. Azure CLI logged in: `az login`
|
||||
2. Environment variables set (or `.env` file in the `python/` directory):
|
||||
```
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com
|
||||
FOUNDRY_MODEL=gpt-4.1
|
||||
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
|
||||
```
|
||||
|
||||
## Samples
|
||||
|
||||
### 01-get-started — Script samples (easy → advanced)
|
||||
|
||||
| # | Sample | Description | Run |
|
||||
|---|--------|-------------|-----|
|
||||
| 01 | [Document Q&A](01-get-started/01_document_qa.py) | Upload a PDF, ask questions with CU-powered extraction | `uv run samples/01-get-started/01_document_qa.py` |
|
||||
| 02 | [Multi-Turn Session](01-get-started/02_multi_turn_session.py) | AgentSession persistence across turns | `uv run samples/01-get-started/02_multi_turn_session.py` |
|
||||
| 03 | [Multi-Modal Chat](01-get-started/03_multimodal_chat.py) | PDF + audio + video parallel analysis | `uv run samples/01-get-started/03_multimodal_chat.py` |
|
||||
| 04 | [Invoice Processing](01-get-started/04_invoice_processing.py) | Structured field extraction with prebuilt-invoice | `uv run samples/01-get-started/04_invoice_processing.py` |
|
||||
| 05 | [Large Doc + file_search](01-get-started/05_large_doc_file_search.py) | CU extraction + OpenAI vector store RAG | `uv run samples/01-get-started/05_large_doc_file_search.py` |
|
||||
|
||||
### 02-devui — Interactive web UI samples
|
||||
|
||||
| # | Sample | Description | Run |
|
||||
|---|--------|-------------|-----|
|
||||
| 01 | [Multi-Modal Agent](02-devui/01-multimodal_agent/) | Web UI for file upload + CU-powered chat | `devui samples/02-devui/01-multimodal_agent` |
|
||||
| 02a | [file_search (Azure OpenAI backend)](02-devui/02-file_search_agent/azure_openai_backend/) | DevUI with CU + Azure OpenAI vector store | `devui samples/02-devui/02-file_search_agent/azure_openai_backend` |
|
||||
| 02b | [file_search (Foundry backend)](02-devui/02-file_search_agent/foundry_backend/) | DevUI with CU + Foundry vector store | `devui samples/02-devui/02-file_search_agent/foundry_backend` |
|
||||
|
||||
## Install (preview)
|
||||
|
||||
```bash
|
||||
pip install --pre agent-framework-azure-contentunderstanding
|
||||
```
|
||||
Binary file not shown.
Reference in New Issue
Block a user