wehub-resource-sync
69 lines
3.4 KiB
C#
69 lines
3.4 KiB
C#
using Microsoft.Agents.AI;
|
|
using Microsoft.Extensions.AI;
|
|
using OpenAI;
|
|
|
|
// ============================================================================
|
|
// Multimodal Agent
|
|
// ============================================================================
|
|
//
|
|
// Vision-capable .NET agent for the Multimodal Attachments demo cell.
|
|
//
|
|
// Design mirrors the LangGraph reference
|
|
// (showcase/integrations/langgraph-python/src/agents/multimodal_agent.py):
|
|
// - Use a vision-capable chat model (gpt-4o / gpt-4o-mini) so images are
|
|
// consumed natively by the model via OpenAI's image content parts.
|
|
// - No tools are registered — the model handles image/PDF analysis directly.
|
|
// - PDF handling: Microsoft.Extensions.AI passes document/data content parts
|
|
// through as DataContent, and modern OpenAI chat models accept PDF input
|
|
// directly. We therefore avoid bundling a PDF extractor (like pypdf on the
|
|
// Python side) and defer to the model's native document handling. If a PDF
|
|
// cannot be read, the model will tell the user — matching the "[Attached
|
|
// document: PDF could not be read.]" graceful degradation in Python.
|
|
//
|
|
// Wire format: `MultimodalEndpoint` parses the modern
|
|
// `{ type: "image" | "document", source: {...} }` content parts CopilotChat
|
|
// emits and forwards them as DataContent parts the chat client can pass to
|
|
// the OpenAI image/file adapters unchanged. The dedicated endpoint exists
|
|
// because the current Microsoft AG-UI ASP.NET adapter rejects content arrays
|
|
// before an AIAgent can see them.
|
|
//
|
|
// Mount point: `/multimodal` (see Program.cs). The Next.js runtime's
|
|
// `src/app/api/copilotkit-multimodal/route.ts` proxies to this endpoint via
|
|
// AG-UI over HTTP.
|
|
//
|
|
// NOTE (harness): unlike the MapAGUI feature agents, this path is wired as a
|
|
// raw `MapPost("/multimodal", ...)` -> MultimodalEndpoint.HandleAsync over a
|
|
// shared IChatClient (SalesAgentFactory.CreateMultimodalChatClient). Because
|
|
// the endpoint consumes a chat client directly (not an AIAgent), there is no
|
|
// AsHarnessAgent transformation here — the harness construction delta does
|
|
// not apply. This factory's `Create` is retained for parity/testing and the
|
|
// `SystemPrompt` const is what MultimodalEndpoint applies as ChatOptions.
|
|
// ============================================================================
|
|
|
|
internal static class MultimodalAgentFactory
|
|
{
|
|
internal const string SystemPrompt =
|
|
"You are a helpful assistant. The user may attach images or documents " +
|
|
"(PDFs). When they do, analyze the attachment carefully and answer the " +
|
|
"user's question. If no attachment is present, answer the text question " +
|
|
"normally. Keep responses concise (1-3 sentences) unless asked to go deep.";
|
|
|
|
public static AIAgent Create(OpenAIClient openAiClient)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(openAiClient);
|
|
|
|
// gpt-4o-mini supports vision natively. Matches the rest of the
|
|
// dotnet showcase (which uses gpt-4o-mini for every cell) so we don't
|
|
// introduce a new model id just for this cell. The LangGraph
|
|
// reference uses gpt-4o for slightly higher image-reasoning quality;
|
|
// gpt-4o-mini is cheaper and still vision-capable.
|
|
var chatClient = openAiClient.GetChatClient("gpt-4o-mini").AsIChatClient();
|
|
|
|
return new ChatClientAgent(
|
|
chatClient,
|
|
name: "MultimodalAgent",
|
|
description: SystemPrompt,
|
|
tools: []);
|
|
}
|
|
}
|