chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates writing custom evaluation functions for domain-specific
|
||||
// checks. Custom evaluators run locally — no cloud evaluator service needed.
|
||||
// For LLM-based quality scoring (relevance, coherence), see Evaluation_SimpleEval.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
AIAgent agent = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are a customer support agent. Help users resolve their issues "
|
||||
+ "politely and provide clear, actionable steps.",
|
||||
name: "SupportAgent");
|
||||
|
||||
// Custom check: the agent should not refuse to help.
|
||||
EvalCheck noRefusal = FunctionEvaluator.Create("no_refusal", (string response) =>
|
||||
!response.Contains("I can't help", StringComparison.OrdinalIgnoreCase)
|
||||
&& !response.Contains("I'm unable to", StringComparison.OrdinalIgnoreCase)
|
||||
&& !response.Contains("outside my scope", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// Custom check: response should include actionable guidance (numbered steps or bullet points).
|
||||
EvalCheck hasActionableSteps = FunctionEvaluator.Create("has_actionable_steps", (string response) =>
|
||||
response.Contains("1.", StringComparison.Ordinal)
|
||||
|| response.Contains("- ", StringComparison.Ordinal)
|
||||
|| response.Contains("• ", StringComparison.Ordinal));
|
||||
|
||||
// Custom check: response should be substantial but not excessively long.
|
||||
EvalCheck reasonableLength = FunctionEvaluator.Create("reasonable_length", (string response) =>
|
||||
response.Length >= 50 && response.Length <= 2000);
|
||||
|
||||
// Combine all custom checks into a local evaluator.
|
||||
LocalEvaluator evaluator = new(noRefusal, hasActionableSteps, reasonableLength);
|
||||
|
||||
string[] queries =
|
||||
[
|
||||
"My order hasn't arrived after two weeks. What should I do?",
|
||||
"I was charged twice for the same item. Can you help?",
|
||||
"How do I return a damaged product?",
|
||||
];
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(queries, evaluator);
|
||||
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
Console.WriteLine();
|
||||
|
||||
for (int i = 0; i < results.Items.Count; i++)
|
||||
{
|
||||
Console.WriteLine($"Query: {queries[i]}");
|
||||
Console.WriteLine($"Response: {(results.InputItems?[i].Response is { } resp ? resp.Substring(0, Math.Min(50, resp.Length)) : "N/A")}...");
|
||||
foreach (var metric in results.Items[i].Metrics)
|
||||
{
|
||||
string status = metric.Value.Interpretation?.Failed == true ? "FAIL" : "PASS";
|
||||
Console.WriteLine($" [{status}] {metric.Key}");
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Evaluation - Custom Evals
|
||||
|
||||
This sample demonstrates writing custom domain-specific evaluation functions using `FunctionEvaluator.Create`. Custom evaluators run locally with no cloud evaluator service needed — useful for enforcing business rules, format requirements, or safety guardrails.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Writing custom checks with `FunctionEvaluator.Create` for domain-specific logic
|
||||
- Checking that a customer support agent doesn't refuse to help
|
||||
- Verifying responses contain actionable steps (numbered lists or bullet points)
|
||||
- Enforcing response length constraints
|
||||
- Combining multiple custom checks into a `LocalEvaluator`
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure authentication available to `DefaultAzureCredential` (for local development, run `az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:FOUNDRY_MODEL="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/02-agents/Evaluation
|
||||
dotnet run --project .\Evaluation_CustomEvals
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation using Foundry quality evaluators (Relevance, Coherence)
|
||||
- [Evaluation_ExpectedOutputs](../Evaluation_ExpectedOutputs/) — Evaluating against ground-truth expected outputs
|
||||
- [Evaluation_MixedProviders](../../../05-end-to-end/Evaluation/Evaluation_MixedProviders/) — Combining custom + Foundry evaluators in one call
|
||||
Reference in New Issue
Block a user