wehub-resource-sync
db620d33df
CodeQL / Analyze (csharp) (push) Waiting to run
CodeQL / Analyze (python) (push) Waiting to run
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
67 lines
2.7 KiB
Python
67 lines
2.7 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
"""DevUI Multi-Modal Agent — file upload + CU-powered analysis.
|
|
|
|
This agent uses Azure Content Understanding to analyze uploaded files
|
|
(PDFs, scanned documents, handwritten images, audio recordings, video)
|
|
and answer questions about them through the DevUI web interface.
|
|
|
|
Unlike the standard azure_responses_agent which sends files directly to the LLM,
|
|
this agent uses CU for structured extraction — superior for scanned PDFs,
|
|
handwritten content, audio transcription, and video analysis.
|
|
|
|
Required environment variables:
|
|
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
|
|
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
|
|
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
|
|
|
|
Run with DevUI:
|
|
uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
|
|
"""
|
|
|
|
import os
|
|
|
|
from agent_framework import Agent
|
|
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
|
|
from azure.core.credentials import AzureKeyCredential
|
|
from azure.identity import AzureCliCredential
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
# --- Auth ---
|
|
_credential = AzureCliCredential()
|
|
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
|
|
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
|
|
|
|
cu = ContentUnderstandingContextProvider(
|
|
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
|
|
credential=_cu_credential,
|
|
# max_wait controls how long before_run() waits for CU analysis before
|
|
# deferring to background. For interactive DevUI use, a short timeout
|
|
# (e.g. 5s) keeps the chat responsive — the agent tells the user the
|
|
# file is still being analyzed and resolves it on the next turn.
|
|
# Use max_wait=None to always wait for analysis to complete.
|
|
max_wait=5.0,
|
|
)
|
|
|
|
client = FoundryChatClient(
|
|
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
|
model=os.environ["FOUNDRY_MODEL"],
|
|
credential=_credential,
|
|
)
|
|
|
|
agent = Agent(
|
|
client=client,
|
|
name="MultiModalDocAgent",
|
|
instructions=(
|
|
"You are a helpful document analysis assistant. "
|
|
"When a user uploads files, they are automatically analyzed using Azure Content Understanding. "
|
|
"Use list_documents() to check which documents are ready, pending, or failed "
|
|
"and to see which files are available for answering questions. "
|
|
"Tell the user if any documents are still being analyzed. "
|
|
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
|
|
"When answering, cite specific content from the documents."
|
|
),
|
|
context_providers=[cu],
|
|
)
|