chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
+16
@@ -0,0 +1,16 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates evaluating a multi-agent workflow against a
|
||||
// golden answer using Foundry's reference-based Similarity evaluator.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
using Microsoft.Extensions.AI;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Build a two-agent workflow: a researcher writes a draft answer, then an
|
||||
// editor polishes it into the final response that we compare to ground truth.
|
||||
// EmitAgentResponseEvents is enabled so the workflow surfaces an AgentResponseEvent
|
||||
// for each agent — this is what EvaluateAsync uses to find the overall final answer.
|
||||
var hostOptions = new AIAgentHostOptions { EmitAgentResponseEvents = true };
|
||||
|
||||
AIAgent researcher = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You research questions and produce a short factual draft answer.",
|
||||
name: "researcher");
|
||||
|
||||
AIAgent editor = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You take a draft answer and produce the final concise response.",
|
||||
name: "editor");
|
||||
|
||||
ExecutorBinding researcherExecutor = researcher.BindAsExecutor(hostOptions);
|
||||
ExecutorBinding editorExecutor = editor.BindAsExecutor(hostOptions);
|
||||
|
||||
Workflow workflow = new WorkflowBuilder(researcherExecutor)
|
||||
.AddEdge(researcherExecutor, editorExecutor)
|
||||
.Build();
|
||||
|
||||
// Run the workflow against the user question.
|
||||
const string Query = "What is the capital of France?";
|
||||
const string GroundTruth = "Paris";
|
||||
|
||||
await using Run run = await InProcessExecution.RunAsync(
|
||||
workflow,
|
||||
new ChatMessage(ChatRole.User, Query));
|
||||
|
||||
// Evaluate the overall workflow output against a golden answer using the
|
||||
// reference-based Similarity evaluator. The 'expectedOutput' value is stamped
|
||||
// onto the overall EvalItem.ExpectedOutput and is surfaced to Foundry as
|
||||
// `ground_truth` in the underlying JSONL payload.
|
||||
//
|
||||
// Per-agent breakdown is disabled here: ground truth applies to the workflow's
|
||||
// final answer, not to each sub-agent's intermediate output. Without
|
||||
// includePerAgent: false, the evaluator would be invoked for per-agent items
|
||||
// (which have no ExpectedOutput) and Similarity would fail validation.
|
||||
FoundryEvals similarity = new(projectClient, deploymentName, FoundryEvals.Similarity);
|
||||
|
||||
AgentEvaluationResults results = await run.EvaluateAsync(
|
||||
similarity,
|
||||
includePerAgent: false,
|
||||
expectedOutput: GroundTruth);
|
||||
|
||||
Console.WriteLine($"Query: {Query}");
|
||||
Console.WriteLine($"Expected: {GroundTruth}");
|
||||
Console.WriteLine($"Provider: {results.ProviderName}");
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($"Report: {results.ReportUrl}");
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Evaluation - Workflow Expected Outputs
|
||||
|
||||
This sample demonstrates evaluating a multi-agent workflow's final answer
|
||||
against a golden expected output using Foundry's reference-based **Similarity**
|
||||
evaluator.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Building a small researcher → editor workflow
|
||||
- Running the workflow and obtaining a `Run`
|
||||
- Calling `run.EvaluateAsync(evaluator, expectedOutput: ...)` to attach a
|
||||
ground-truth answer to the overall workflow item
|
||||
- Using `FoundryEvals.Similarity`, which requires a `ground_truth` value
|
||||
per item
|
||||
|
||||
The `expectedOutput` value is stamped onto the overall `EvalItem.ExpectedOutput`
|
||||
and is surfaced to Foundry as `ground_truth` in the JSONL payload sent to the
|
||||
Evals API.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure authentication available to `DefaultAzureCredential` (for local development, run `az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:FOUNDRY_MODEL="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/03-workflows/Evaluation
|
||||
dotnet run --project .\Evaluation_WorkflowExpectedOutputs
|
||||
```
|
||||
Reference in New Issue
Block a user