chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:25 +08:00
commit db620d33df
5151 changed files with 925932 additions and 0 deletions
@@ -0,0 +1,314 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
using static Microsoft.Agents.AI.UnitTests.LoopTestHelpers;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="AIJudgeLoopEvaluator"/> class.
/// </summary>
public class AIJudgeLoopEvaluatorTests
{
/// <summary>
/// Verify that the evaluator stops when the judge reports the request was answered.
/// </summary>
[Fact]
public async Task EvaluateAsync_Answered_StopsAsync()
{
// Arrange
var judgeClient = CreateJudgeClient("{\"answered\":true}");
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that when not answered the evaluator continues with feedback carrying the judge's gap analysis.
/// </summary>
[Fact]
public async Task EvaluateAsync_NotAnswered_ContinuesWithGapAnalysisAsync()
{
// Arrange
var judgeClient = CreateJudgeClient("{\"answered\":false,\"gapAnalysis\":\"the cost estimate is missing\"}");
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.NotNull(evaluation.Feedback);
Assert.Contains("the cost estimate is missing", evaluation.Feedback!);
Assert.DoesNotContain(AIJudgeLoopEvaluator.GapAnalysisPlaceholder, evaluation.Feedback!);
}
/// <summary>
/// Verify that the evaluator falls back to text parsing and stops when the DONE verdict marker is present.
/// </summary>
[Fact]
public async Task EvaluateAsync_TextFallback_StopsWhenAnsweredAsync()
{
// Arrange
var judgeClient = CreateJudgeClient(AIJudgeLoopEvaluator.DoneVerdictMarker);
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that the gap-analysis placeholder is filled with a fallback token when no structured output is produced.
/// </summary>
[Fact]
public async Task EvaluateAsync_NotAnswered_TextFallback_InjectsUnknownGapAnalysisAsync()
{
// Arrange
var judgeClient = CreateJudgeClient(AIJudgeLoopEvaluator.MoreVerdictMarker);
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Contains("<unknown>", evaluation.Feedback!);
}
/// <summary>
/// Verify that the text fallback keeps looping for replies that merely contain the substring "ANSWERED" (for
/// example "UNANSWERED" or "NOT ANSWERED") rather than the explicit DONE verdict marker.
/// </summary>
[Theory]
[InlineData("UNANSWERED")]
[InlineData("NOT ANSWERED")]
[InlineData("The request is not yet answered.")]
public async Task EvaluateAsync_TextFallback_AmbiguousReply_ContinuesAsync(string reply)
{
// Arrange
var judgeClient = CreateJudgeClient(reply);
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that custom judge instructions from options are sent to the judge client.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomInstructions_AreSentToJudgeAsync()
{
// Arrange
List<ChatMessage>? judgeMessages = null;
var judgeMock = new Mock<IChatClient>();
judgeMock.Setup(c => c.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((msgs, _, _) => judgeMessages = msgs.ToList())
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "{\"answered\":true}")));
var evaluator = new AIJudgeLoopEvaluator(judgeMock.Object, new AIJudgeLoopEvaluatorOptions { Instructions = "CUSTOM JUDGE PROMPT" });
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
Assert.NotNull(judgeMessages);
Assert.Contains(judgeMessages!, m => m.Role == ChatRole.System && m.Text == "CUSTOM JUDGE PROMPT");
}
/// <summary>
/// Verify that a custom feedback message template from options is honored.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomFeedbackMessageTemplate_IsHonoredAsync()
{
// Arrange
var judgeClient = CreateJudgeClient("{\"answered\":false,\"gapAnalysis\":\"add unit tests\"}");
const string Template = "Please address: " + AIJudgeLoopEvaluator.GapAnalysisPlaceholder;
var evaluator = new AIJudgeLoopEvaluator(judgeClient, new AIJudgeLoopEvaluatorOptions { FeedbackMessageTemplate = Template });
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.Equal("Please address: add unit tests", evaluation.Feedback);
}
/// <summary>
/// Verify that non-text content in the original request (for example an image) is forwarded to the judge
/// rather than being silently dropped when flattening the request to text.
/// </summary>
[Fact]
public async Task EvaluateAsync_NonTextRequestContent_IsForwardedToJudgeAsync()
{
// Arrange
List<ChatMessage>? judgeMessages = null;
var judgeMock = new Mock<IChatClient>();
judgeMock.Setup(c => c.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((msgs, _, _) => judgeMessages = msgs.ToList())
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "{\"answered\":true}")));
var evaluator = new AIJudgeLoopEvaluator(judgeMock.Object);
var imageContent = new DataContent(new byte[] { 1, 2, 3, 4 }, "image/png");
var context = new LoopContext(
new Mock<AIAgent>().Object,
new ChatClientAgentSession(),
[new ChatMessage(ChatRole.User, [imageContent])],
new AgentResponse([new ChatMessage(ChatRole.Assistant, "partial answer")]));
// Act
await evaluator.EvaluateAsync(context);
// Assert
Assert.NotNull(judgeMessages);
ChatMessage userMessage = Assert.Single(judgeMessages!, m => m.Role == ChatRole.User);
Assert.Contains(userMessage.Contents.OfType<DataContent>(), c => c.MediaType == "image/png");
}
/// <summary>
/// Verify that the constructor throws when the judge client is null.
/// </summary>
[Fact]
public void AIJudgeLoopEvaluator_NullClient_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("judgeClient", () => new AIJudgeLoopEvaluator(null!));
}
/// <summary>
/// Verify that EvaluateAsync throws when the context is null.
/// </summary>
[Fact]
public async Task EvaluateAsync_NullContext_ThrowsAsync()
{
// Arrange
var evaluator = new AIJudgeLoopEvaluator(CreateJudgeClient("{\"answered\":true}"));
// Act & Assert
await Assert.ThrowsAsync<ArgumentNullException>("context", async () => await evaluator.EvaluateAsync(null!));
}
/// <summary>
/// Verify that supplied criteria are rendered into the default judge instructions as a bullet list and the
/// placeholder is consumed.
/// </summary>
[Fact]
public async Task EvaluateAsync_Criteria_AreRenderedIntoDefaultInstructionsAsync()
{
// Arrange
var judgeClient = CreateCapturingJudgeClient("{\"answered\":true}", out List<ChatMessage> judgeMessages);
var options = new AIJudgeLoopEvaluatorOptions { Criteria = ["Must cite sources", "Must be under 200 words"] };
var evaluator = new AIJudgeLoopEvaluator(judgeClient, options);
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
string system = judgeMessages.Single(static m => m.Role == ChatRole.System).Text;
Assert.Contains("The response must satisfy all of the following criteria:", system);
Assert.Contains("- Must cite sources", system);
Assert.Contains("- Must be under 200 words", system);
Assert.DoesNotContain(AIJudgeLoopEvaluator.CriteriaPlaceholder, system);
}
/// <summary>
/// Verify that when no criteria are supplied the placeholder is removed and no criteria block is added to the
/// default instructions.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoCriteria_LeavesDefaultInstructionsWithoutCriteriaBlockAsync()
{
// Arrange
var judgeClient = CreateCapturingJudgeClient("{\"answered\":true}", out List<ChatMessage> judgeMessages);
var evaluator = new AIJudgeLoopEvaluator(judgeClient);
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
string system = judgeMessages.Single(static m => m.Role == ChatRole.System).Text;
Assert.DoesNotContain(AIJudgeLoopEvaluator.CriteriaPlaceholder, system);
Assert.DoesNotContain("The response must satisfy all of the following criteria:", system);
}
/// <summary>
/// Verify that criteria are injected at the placeholder location in custom instructions.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomInstructionsWithPlaceholder_InjectsCriteriaAsync()
{
// Arrange
var judgeClient = CreateCapturingJudgeClient("{\"answered\":true}", out List<ChatMessage> judgeMessages);
const string Instructions = "Judge the answer." + AIJudgeLoopEvaluator.CriteriaPlaceholder + " Be strict.";
var options = new AIJudgeLoopEvaluatorOptions { Instructions = Instructions, Criteria = ["Must include code"] };
var evaluator = new AIJudgeLoopEvaluator(judgeClient, options);
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
string system = judgeMessages.Single(static m => m.Role == ChatRole.System).Text;
Assert.StartsWith("Judge the answer.", system);
Assert.EndsWith("Be strict.", system);
Assert.Contains("- Must include code", system);
Assert.DoesNotContain(AIJudgeLoopEvaluator.CriteriaPlaceholder, system);
}
/// <summary>
/// Verify that custom instructions without the placeholder do not receive the criteria.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomInstructionsWithoutPlaceholder_OmitsCriteriaAsync()
{
// Arrange
var judgeClient = CreateCapturingJudgeClient("{\"answered\":true}", out List<ChatMessage> judgeMessages);
const string Instructions = "Judge the answer and be strict.";
var options = new AIJudgeLoopEvaluatorOptions { Instructions = Instructions, Criteria = ["Must include code"] };
var evaluator = new AIJudgeLoopEvaluator(judgeClient, options);
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
string system = judgeMessages.Single(static m => m.Role == ChatRole.System).Text;
Assert.Equal(Instructions, system);
}
private static LoopContext CreateContext() => new(
new Mock<AIAgent>().Object,
new ChatClientAgentSession(),
[new ChatMessage(ChatRole.User, "original question")],
new AgentResponse([new ChatMessage(ChatRole.Assistant, "partial answer")]));
}
@@ -0,0 +1,319 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
using Moq.Protected;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="BackgroundTaskCompletionLoopEvaluator"/> class.
/// </summary>
public class BackgroundTaskCompletionLoopEvaluatorTests
{
/// <summary>
/// Verify that the constructor succeeds with no options and with a custom template.
/// </summary>
[Fact]
public void BackgroundTaskCompletionLoopEvaluator_ValidConstruction_Succeeds()
{
// Act & Assert
_ = new BackgroundTaskCompletionLoopEvaluator();
_ = new BackgroundTaskCompletionLoopEvaluator(new BackgroundTaskCompletionLoopEvaluatorOptions { FeedbackMessageTemplate = "custom" });
}
/// <summary>
/// Verify that evaluation throws when no <see cref="BackgroundAgentsProvider"/> can be resolved from the agent.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoBackgroundAgentsProvider_ThrowsAsync()
{
// Arrange — a bare agent that resolves no providers.
var evaluator = new BackgroundTaskCompletionLoopEvaluator();
var context = CreateContext(new Mock<AIAgent>().Object, new ChatClientAgentSession());
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(async () => await evaluator.EvaluateAsync(context));
}
/// <summary>
/// Verify that the evaluator continues while a background task is still running and that the feedback lists the
/// running task and its count.
/// </summary>
[Fact]
public async Task EvaluateAsync_RunningTask_ContinuesWithFeedbackAsync()
{
// Arrange
var tcs = new TaskCompletionSource<AgentResponse>();
var backgroundAgent = CreateMockAgentWithRunResult("Research", tcs.Task);
var provider = new BackgroundAgentsProvider(new[] { backgroundAgent });
var session = new ChatClientAgentSession();
IEnumerable<AITool> tools = await CreateToolsForSessionAsync(provider, session);
await StartTaskAsync(tools, "Research", "Find information about AI", "Research AI topics");
AIAgent agent = CreateAgent(provider);
var evaluator = new BackgroundTaskCompletionLoopEvaluator();
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.NotNull(evaluation.Feedback);
Assert.Contains("1 background task(s)", evaluation.Feedback!);
Assert.Contains("#1", evaluation.Feedback!);
Assert.Contains("Research", evaluation.Feedback!);
Assert.Contains("Research AI topics", evaluation.Feedback!);
// Cleanup — complete the in-flight task to avoid leaking a pending task.
tcs.SetResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "done")));
}
/// <summary>
/// Verify that the evaluator stops once every background task has reached a terminal state.
/// </summary>
[Fact]
public async Task EvaluateAsync_AllTasksTerminal_StopsAsync()
{
// Arrange
var tcs = new TaskCompletionSource<AgentResponse>();
var backgroundAgent = CreateMockAgentWithRunResult("Research", tcs.Task);
var provider = new BackgroundAgentsProvider(new[] { backgroundAgent });
var session = new ChatClientAgentSession();
IEnumerable<AITool> tools = await CreateToolsForSessionAsync(provider, session);
await StartTaskAsync(tools, "Research", "Task 1", "First task");
// Complete the task and wait for it to be finalized.
tcs.SetResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "Result 1")));
await WaitForCompletionAsync(tools, 1);
AIAgent agent = CreateAgent(provider);
var evaluator = new BackgroundTaskCompletionLoopEvaluator();
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
/// <summary>
/// Verify that the evaluator stops when no background tasks have been started.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoTasks_StopsAsync()
{
// Arrange
var backgroundAgent = CreateMockAgent("Research", "Research agent");
var provider = new BackgroundAgentsProvider(new[] { backgroundAgent });
var session = new ChatClientAgentSession();
_ = await CreateToolsForSessionAsync(provider, session);
AIAgent agent = CreateAgent(provider);
var evaluator = new BackgroundTaskCompletionLoopEvaluator();
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
/// <summary>
/// Verify that a custom feedback template is honored, including the running task list placeholder.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomTemplate_IsHonoredAsync()
{
// Arrange
var tcs = new TaskCompletionSource<AgentResponse>();
var backgroundAgent = CreateMockAgentWithRunResult("Research", tcs.Task);
var provider = new BackgroundAgentsProvider(new[] { backgroundAgent });
var session = new ChatClientAgentSession();
IEnumerable<AITool> tools = await CreateToolsForSessionAsync(provider, session);
await StartTaskAsync(tools, "Research", "Task 1", "First task");
AIAgent agent = CreateAgent(provider);
var options = new BackgroundTaskCompletionLoopEvaluatorOptions
{
FeedbackMessageTemplate = "Pending:\n" + BackgroundTaskCompletionLoopEvaluator.IncompleteTasksPlaceholder,
};
var evaluator = new BackgroundTaskCompletionLoopEvaluator(options);
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.StartsWith("Pending:", evaluation.Feedback);
Assert.Contains("#1", evaluation.Feedback!);
Assert.Contains("First task", evaluation.Feedback!);
// Cleanup.
tcs.SetResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "done")));
}
/// <summary>
/// Verify that <see cref="BackgroundAgentsProvider.GetIncompleteTasks"/> returns only the tasks that are still
/// running, excluding completed ones.
/// </summary>
[Fact]
public async Task GetIncompleteTasks_ReturnsOnlyRunningTasksAsync()
{
// Arrange — two tasks, one of which completes.
var tcs1 = new TaskCompletionSource<AgentResponse>();
var tcs2 = new TaskCompletionSource<AgentResponse>();
var agent1 = CreateMockAgentWithRunResult("Research", tcs1.Task);
var agent2 = CreateMockAgentWithRunResult("Writer", tcs2.Task);
var provider = new BackgroundAgentsProvider(new[] { agent1, agent2 });
var session = new ChatClientAgentSession();
IEnumerable<AITool> tools = await CreateToolsForSessionAsync(provider, session);
await StartTaskAsync(tools, "Research", "Task 1", "First task");
await StartTaskAsync(tools, "Writer", "Task 2", "Second task");
// Complete only the first task and wait for it to be finalized.
tcs1.SetResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "Result 1")));
await WaitForCompletionAsync(tools, 1);
// Act
IReadOnlyList<BackgroundTaskInfo> incomplete = provider.GetIncompleteTasks(session);
// Assert — only the still-running second task remains.
BackgroundTaskInfo task = Assert.Single(incomplete);
Assert.Equal(2, task.Id);
Assert.Equal("Writer", task.AgentName);
Assert.Equal(BackgroundTaskStatus.Running, task.Status);
// Cleanup.
tcs2.SetResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "done")));
}
/// <summary>
/// Verify that a task persisted as <see cref="BackgroundTaskStatus.Running"/> but with no corresponding in-flight
/// runtime reference (as happens after a session is serialized and restored) is treated as
/// <see cref="BackgroundTaskStatus.Lost"/> and excluded from the incomplete set, so the loop does not spin forever.
/// </summary>
[Fact]
public async Task GetIncompleteTasks_TaskWithNoInFlightReference_IsTreatedAsLostAndExcludedAsync()
{
// Arrange — seed a Running task into session state without any runtime in-flight reference, simulating a restore.
var backgroundAgent = CreateMockAgent("Research", "Research agent");
var provider = new BackgroundAgentsProvider(new[] { backgroundAgent });
var session = new ChatClientAgentSession();
SeedRunningTaskWithoutInFlight(session, 1, "Research", "First task");
// Act — querying refreshes task state, which finalizes the orphaned task to Lost.
IReadOnlyList<BackgroundTaskInfo> incomplete = provider.GetIncompleteTasks(session);
// Assert — the lost task is not returned, and the evaluator stops rather than looping forever.
Assert.Empty(incomplete);
AIAgent agent = CreateAgent(provider);
var evaluator = new BackgroundTaskCompletionLoopEvaluator();
LoopEvaluation evaluation = await evaluator.EvaluateAsync(CreateContext(agent, session));
Assert.False(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
private static ChatClientAgent CreateAgent(params AIContextProvider[] providers)
{
var chatClient = new Mock<IChatClient>().Object;
return new ChatClientAgent(chatClient, new ChatClientAgentOptions { AIContextProviders = providers });
}
private static LoopContext CreateContext(AIAgent agent, AgentSession session) => new(
agent,
session,
[new ChatMessage(ChatRole.User, "do the work")],
new AgentResponse([new ChatMessage(ChatRole.Assistant, "in progress")]));
private static async Task<IEnumerable<AITool>> CreateToolsForSessionAsync(BackgroundAgentsProvider provider, AgentSession session)
{
var mockAgent = new Mock<AIAgent>().Object;
#pragma warning disable MAAI001
var context = new AIContextProvider.InvokingContext(mockAgent, session, new AIContext());
#pragma warning restore MAAI001
AIContext result = await provider.InvokingAsync(context);
return result.Tools!;
}
private static async Task StartTaskAsync(IEnumerable<AITool> tools, string agentName, string input, string description)
{
AIFunction startTask = GetTool(tools, "background_agents_start_task");
await startTask.InvokeAsync(new AIFunctionArguments
{
["agentName"] = agentName,
["input"] = input,
["description"] = description,
});
}
private static async Task WaitForCompletionAsync(IEnumerable<AITool> tools, int taskId)
{
AIFunction waitForFirst = GetTool(tools, "background_agents_wait_for_first_completion");
await waitForFirst.InvokeAsync(new AIFunctionArguments
{
["taskIds"] = new List<int> { taskId },
});
}
private static void SeedRunningTaskWithoutInFlight(AgentSession session, int id, string agentName, string description)
{
var state = new BackgroundAgentState { NextTaskId = id + 1 };
state.Tasks.Add(new BackgroundTaskInfo
{
Id = id,
AgentName = agentName,
Description = description,
Status = BackgroundTaskStatus.Running,
});
// Persist under the BackgroundAgentsProvider's state key with no runtime in-flight entry, mirroring a restored
// session whose in-flight task references have been lost.
session.StateBag.SetValue(nameof(BackgroundAgentsProvider), state, AgentJsonUtilities.DefaultOptions);
}
private static AIAgent CreateMockAgent(string? name, string? description)
{
var mock = new Mock<AIAgent>();
mock.SetupGet(a => a.Name).Returns(name!);
mock.SetupGet(a => a.Description).Returns(description);
return mock.Object;
}
private static AIAgent CreateMockAgentWithRunResult(string name, Task<AgentResponse> result)
{
var mock = new Mock<AIAgent>();
mock.SetupGet(a => a.Name).Returns(name);
mock.Protected()
.Setup<ValueTask<AgentSession>>(
"CreateSessionCoreAsync",
ItExpr.IsAny<CancellationToken>())
.Returns(new ValueTask<AgentSession>(new ChatClientAgentSession()));
mock.Protected()
.Setup<Task<AgentResponse>>(
"RunCoreAsync",
ItExpr.IsAny<IEnumerable<ChatMessage>>(),
ItExpr.IsAny<AgentSession>(),
ItExpr.IsAny<AgentRunOptions>(),
ItExpr.IsAny<CancellationToken>())
.Returns(result);
return mock.Object;
}
private static AIFunction GetTool(IEnumerable<AITool> tools, string name)
{
return (AIFunction)tools.First(t => t is AIFunction f && f.Name == name);
}
}
@@ -0,0 +1,145 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="CompletionMarkerLoopEvaluator"/> class.
/// </summary>
public class CompletionMarkerLoopEvaluatorTests
{
/// <summary>
/// Verify that the constructor throws when the marker is null, empty, or whitespace.
/// </summary>
/// <param name="marker">The invalid marker value.</param>
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void CompletionMarkerLoopEvaluator_InvalidMarker_Throws(string? marker)
{
// Act & Assert
Assert.ThrowsAny<ArgumentException>(() => new CompletionMarkerLoopEvaluator(marker!));
}
/// <summary>
/// Verify that the evaluator stops the loop when the marker appears in the latest response.
/// </summary>
[Fact]
public async Task EvaluateAsync_MarkerPresent_StopsAsync()
{
// Arrange
var evaluator = new CompletionMarkerLoopEvaluator("DONE");
LoopContext context = CreateContext("all DONE here");
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that the evaluator continues with default feedback (containing the marker) when the marker is absent.
/// </summary>
[Fact]
public async Task EvaluateAsync_MarkerAbsent_ContinuesWithDefaultFeedbackAsync()
{
// Arrange
var evaluator = new CompletionMarkerLoopEvaluator("DONE");
LoopContext context = CreateContext("still working");
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.NotNull(evaluation.Feedback);
Assert.Contains("DONE", evaluation.Feedback!);
Assert.DoesNotContain(CompletionMarkerLoopEvaluator.CompletionMarkerPlaceholder, evaluation.Feedback!);
}
/// <summary>
/// Verify that a custom feedback template is honored, with the completion marker substituted for the placeholder.
/// </summary>
[Fact]
public async Task EvaluateAsync_MarkerAbsent_CustomTemplate_IsHonoredAsync()
{
// Arrange
const string Template = "Keep going and finish with " + CompletionMarkerLoopEvaluator.CompletionMarkerPlaceholder + " when done.";
var evaluator = new CompletionMarkerLoopEvaluator("FINISHED", new CompletionMarkerLoopEvaluatorOptions { FeedbackMessageTemplate = Template });
LoopContext context = CreateContext("still working");
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Equal("Keep going and finish with FINISHED when done.", evaluation.Feedback);
}
/// <summary>
/// Verify that a custom feedback template containing the last-response placeholder echoes the agent's latest
/// response text, with no leftover placeholder.
/// </summary>
[Fact]
public async Task EvaluateAsync_MarkerAbsent_CustomTemplate_SubstitutesLastResponseAsync()
{
// Arrange
const string Template = "Your previous attempt was: '" + CompletionMarkerLoopEvaluator.LastResponsePlaceholder +
"'. Improve it and finish with " + CompletionMarkerLoopEvaluator.CompletionMarkerPlaceholder + " when done.";
var evaluator = new CompletionMarkerLoopEvaluator("FINISHED", new CompletionMarkerLoopEvaluatorOptions { FeedbackMessageTemplate = Template });
LoopContext context = CreateContext("candidate name: NoteNest");
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Equal("Your previous attempt was: 'candidate name: NoteNest'. Improve it and finish with FINISHED when done.", evaluation.Feedback);
Assert.DoesNotContain(CompletionMarkerLoopEvaluator.LastResponsePlaceholder, evaluation.Feedback!);
}
/// <summary>
/// Verify that the default feedback template does not include the agent's latest response text (the last-response
/// placeholder is opt-in via a custom template).
/// </summary>
[Fact]
public async Task EvaluateAsync_MarkerAbsent_DefaultTemplate_DoesNotIncludeLastResponseAsync()
{
// Arrange
var evaluator = new CompletionMarkerLoopEvaluator("DONE");
LoopContext context = CreateContext("candidate name: NoteNest");
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Equal(CompletionMarkerLoopEvaluator.DefaultFeedbackMessageTemplate.Replace(CompletionMarkerLoopEvaluator.CompletionMarkerPlaceholder, "DONE"), evaluation.Feedback);
Assert.DoesNotContain("NoteNest", evaluation.Feedback!);
}
/// <summary>
/// Verify that EvaluateAsync throws when the context is null.
/// </summary>
[Fact]
public async Task EvaluateAsync_NullContext_ThrowsAsync()
{
// Arrange
var evaluator = new CompletionMarkerLoopEvaluator("DONE");
// Act & Assert
await Assert.ThrowsAsync<ArgumentNullException>("context", async () => await evaluator.EvaluateAsync(null!));
}
private static LoopContext CreateContext(string responseText) => new(
new Mock<AIAgent>().Object,
new ChatClientAgentSession(),
[new ChatMessage(ChatRole.User, "go")],
new AgentResponse([new ChatMessage(ChatRole.Assistant, responseText)]));
}
@@ -0,0 +1,113 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="DelegateLoopEvaluator"/> class.
/// </summary>
public class DelegateLoopEvaluatorTests
{
/// <summary>
/// Verify that the constructor throws when the evaluate delegate is null.
/// </summary>
[Fact]
public void DelegateLoopEvaluator_NullDelegate_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("evaluate", () => new DelegateLoopEvaluator(null!));
}
/// <summary>
/// Verify that EvaluateAsync throws when the context is null.
/// </summary>
[Fact]
public async Task EvaluateAsync_NullContext_ThrowsAsync()
{
// Arrange
var evaluator = new DelegateLoopEvaluator((_, _) => new ValueTask<LoopEvaluation>(LoopEvaluation.Stop()));
// Act & Assert
await Assert.ThrowsAsync<ArgumentNullException>("context", async () => await evaluator.EvaluateAsync(null!));
}
/// <summary>
/// Verify that EvaluateAsync invokes the supplied delegate and returns the evaluation it produces.
/// </summary>
[Fact]
public async Task EvaluateAsync_InvokesDelegate_AndReturnsItsEvaluationAsync()
{
// Arrange
bool invoked = false;
var expected = LoopEvaluation.Continue("feedback");
var evaluator = new DelegateLoopEvaluator((_, _) =>
{
invoked = true;
return new ValueTask<LoopEvaluation>(expected);
});
LoopContext context = CreateContext();
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(invoked);
Assert.Same(expected, evaluation);
}
/// <summary>
/// Verify that EvaluateAsync passes the same context instance to the delegate.
/// </summary>
[Fact]
public async Task EvaluateAsync_PassesContextToDelegateAsync()
{
// Arrange
LoopContext? received = null;
var evaluator = new DelegateLoopEvaluator((ctx, _) =>
{
received = ctx;
return new ValueTask<LoopEvaluation>(LoopEvaluation.Stop());
});
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context);
// Assert
Assert.Same(context, received);
}
/// <summary>
/// Verify that EvaluateAsync forwards the cancellation token to the delegate.
/// </summary>
[Fact]
public async Task EvaluateAsync_ForwardsCancellationTokenToDelegateAsync()
{
// Arrange
using var cts = new CancellationTokenSource();
CancellationToken received = default;
var evaluator = new DelegateLoopEvaluator((_, ct) =>
{
received = ct;
return new ValueTask<LoopEvaluation>(LoopEvaluation.Stop());
});
LoopContext context = CreateContext();
// Act
await evaluator.EvaluateAsync(context, cts.Token);
// Assert
Assert.Equal(cts.Token, received);
}
private static LoopContext CreateContext() => new(
new Mock<AIAgent>().Object,
new ChatClientAgentSession(),
[new ChatMessage(ChatRole.User, "go")],
new AgentResponse([new ChatMessage(ChatRole.Assistant, "response")]));
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,146 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="LoopContext"/> class, including its public constructor used to test custom evaluators.
/// </summary>
public class LoopContextTests
{
/// <summary>
/// Verify that the constructor throws when the agent is null.
/// </summary>
[Fact]
public void Constructor_NullAgent_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("agent", () => new LoopContext(
null!, new ChatClientAgentSession(), [], CreateResponse()));
}
/// <summary>
/// Verify that the constructor throws when the session is null.
/// </summary>
[Fact]
public void Constructor_NullSession_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("session", () => new LoopContext(
new Mock<AIAgent>().Object, null!, [], CreateResponse()));
}
/// <summary>
/// Verify that the constructor throws when the initial messages are null.
/// </summary>
[Fact]
public void Constructor_NullInitialMessages_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("initialMessages", () => new LoopContext(
new Mock<AIAgent>().Object, new ChatClientAgentSession(), null!, CreateResponse()));
}
/// <summary>
/// Verify that the constructor throws when the last response is null.
/// </summary>
[Fact]
public void Constructor_NullLastResponse_Throws()
{
// Act & Assert
Assert.Throws<ArgumentNullException>("lastResponse", () => new LoopContext(
new Mock<AIAgent>().Object, new ChatClientAgentSession(), [], null!));
}
/// <summary>
/// Verify that the constructor populates the properties and that LastResponse is never null.
/// </summary>
[Fact]
public void Constructor_ValidArguments_SetsProperties()
{
// Arrange
var agent = new Mock<AIAgent>().Object;
var session = new ChatClientAgentSession();
ChatMessage[] initialMessages = [new ChatMessage(ChatRole.User, "go")];
var response = CreateResponse("done");
// Act
var context = new LoopContext(agent, session, initialMessages, response);
// Assert
Assert.Same(agent, context.Agent);
Assert.Same(session, context.Session);
Assert.Same(initialMessages, context.InitialMessages);
Assert.Same(response, context.LastResponse);
Assert.Null(context.RunOptions);
Assert.NotNull(context.AdditionalProperties);
Assert.Equal(0, context.Iteration);
Assert.Empty(context.Feedback);
}
/// <summary>
/// Verify that the session can be replaced through the internal setter (used by the loop for fresh contexts).
/// </summary>
[Fact]
public void Session_IsInternallySettable()
{
// Arrange
var context = new LoopContext(
new Mock<AIAgent>().Object, new ChatClientAgentSession(), [], CreateResponse());
var newSession = new ChatClientAgentSession();
// Act
context.Session = newSession;
// Assert
Assert.Same(newSession, context.Session);
}
/// <summary>
/// Verify that <see cref="LoopContext.Feedback"/> can be assigned through its internal setter.
/// </summary>
[Fact]
public void Feedback_IsInternallySettable()
{
// Arrange
var context = new LoopContext(
new Mock<AIAgent>().Object, new ChatClientAgentSession(), [], CreateResponse());
// Act
context.Feedback = ["first", null];
// Assert
Assert.Equal(["first", null], context.Feedback);
}
/// <summary>
/// Verify that an evaluator can be evaluated against a publicly-constructed context (the scenario the public
/// constructor exists to support).
/// </summary>
[Fact]
public async Task PubliclyConstructedContext_CanEvaluateEvaluatorAsync()
{
// Arrange
var context = new LoopContext(
new Mock<AIAgent>().Object,
new ChatClientAgentSession(),
[new ChatMessage(ChatRole.User, "go")],
CreateResponse("done"));
var evaluator = new DelegateLoopEvaluator((ctx, _) =>
new ValueTask<LoopEvaluation>(
ctx.LastResponse.Text == "done" ? LoopEvaluation.Stop() : LoopEvaluation.Continue()));
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
private static AgentResponse CreateResponse(string text = "response") =>
new([new ChatMessage(ChatRole.Assistant, text)]);
}
@@ -0,0 +1,55 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="LoopEvaluation"/> class.
/// </summary>
public class LoopEvaluationTests
{
/// <summary>
/// Verify that Stop produces an evaluation that does not re-invoke and carries no feedback.
/// </summary>
[Fact]
public void Stop_DoesNotReinvoke_AndHasNoFeedback()
{
// Act
var evaluation = LoopEvaluation.Stop();
// Assert
Assert.False(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
/// <summary>
/// Verify that Continue with no argument re-invokes and carries no feedback.
/// </summary>
[Fact]
public void Continue_NoFeedback_ReinvokesWithNullFeedback()
{
// Act
var evaluation = LoopEvaluation.Continue();
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
/// <summary>
/// Verify that Continue with whitespace-only feedback normalizes the feedback to null, matching the documented
/// "null, empty, or whitespace is treated as no feedback" semantics.
/// </summary>
[Theory]
[InlineData("")]
[InlineData(" ")]
[InlineData("\t\n")]
public void Continue_WhitespaceFeedback_NormalizesToNull(string feedback)
{
// Act
var evaluation = LoopEvaluation.Continue(feedback);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.Null(evaluation.Feedback);
}
}
@@ -0,0 +1,141 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
using Moq.Protected;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Shared helpers used by the LoopAgent and LoopEvaluator unit tests.
/// </summary>
internal static class LoopTestHelpers
{
/// <summary>
/// Creates a <see cref="DelegateLoopEvaluator"/> that re-invokes the agent (without feedback) while the
/// supplied predicate returns <see langword="true"/>.
/// </summary>
public static DelegateLoopEvaluator While(Func<LoopContext, bool> shouldReinvoke) =>
new((context, _) =>
new ValueTask<LoopEvaluation>(
shouldReinvoke(context) ? LoopEvaluation.Continue() : LoopEvaluation.Stop()));
/// <summary>
/// Creates a mocked judge <see cref="IChatClient"/> that always returns the supplied response text.
/// </summary>
public static IChatClient CreateJudgeClient(string responseText)
{
var mock = new Mock<IChatClient>();
mock.Setup(c => c.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, responseText)));
return mock.Object;
}
/// <summary>
/// Creates a mocked judge <see cref="IChatClient"/> that always returns the supplied response text and captures the
/// messages it was invoked with via <paramref name="capturedMessages"/>.
/// </summary>
public static IChatClient CreateCapturingJudgeClient(string responseText, out List<ChatMessage> capturedMessages)
{
var captured = new List<ChatMessage>();
capturedMessages = captured;
var mock = new Mock<IChatClient>();
mock.Setup(c => c.GetResponseAsync(
It.IsAny<IEnumerable<ChatMessage>>(),
It.IsAny<ChatOptions>(),
It.IsAny<CancellationToken>()))
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((messages, _, _) =>
{
captured.Clear();
captured.AddRange(messages);
})
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, responseText)));
return mock.Object;
}
public static async IAsyncEnumerable<T> ToAsyncEnumerableAsync<T>(
IEnumerable<T> items,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var item in items)
{
cancellationToken.ThrowIfCancellationRequested();
yield return item;
await Task.Yield();
}
}
}
/// <summary>
/// Captures the messages sent to a mocked non-streaming inner agent and produces responses by call index.
/// </summary>
internal sealed class InnerAgentCapture
{
public InnerAgentCapture(Func<int, AgentResponse> responseFactory)
{
this.Mock
.Protected()
.Setup<Task<AgentResponse>>("RunCoreAsync",
ItExpr.IsAny<IEnumerable<ChatMessage>>(),
ItExpr.IsAny<AgentSession?>(),
ItExpr.IsAny<AgentRunOptions?>(),
ItExpr.IsAny<CancellationToken>())
.Callback<IEnumerable<ChatMessage>, AgentSession?, AgentRunOptions?, CancellationToken>((msgs, session, _, _) =>
{
this.CallCount++;
this.MessagesPerCall.Add(msgs.ToList());
this.SessionsPerCall.Add(session);
})
.ReturnsAsync(() => responseFactory(this.CallCount));
}
public Mock<AIAgent> Mock { get; } = new();
public AIAgent Agent => this.Mock.Object;
public int CallCount { get; private set; }
public List<List<ChatMessage>> MessagesPerCall { get; } = [];
public List<AgentSession?> SessionsPerCall { get; } = [];
}
/// <summary>
/// Captures the messages sent to a mocked streaming inner agent and produces updates by call index.
/// </summary>
internal sealed class InnerStreamingCapture
{
public InnerStreamingCapture(Func<int, AgentResponseUpdate[]> updatesFactory)
{
this.Mock
.Protected()
.Setup<IAsyncEnumerable<AgentResponseUpdate>>("RunCoreStreamingAsync",
ItExpr.IsAny<IEnumerable<ChatMessage>>(),
ItExpr.IsAny<AgentSession?>(),
ItExpr.IsAny<AgentRunOptions?>(),
ItExpr.IsAny<CancellationToken>())
.Returns<IEnumerable<ChatMessage>, AgentSession?, AgentRunOptions?, CancellationToken>((msgs, _, _, ct) =>
{
this.CallCount++;
this.MessagesPerCall.Add(msgs.ToList());
return LoopTestHelpers.ToAsyncEnumerableAsync(updatesFactory(this.CallCount), ct);
});
}
public Mock<AIAgent> Mock { get; } = new();
public AIAgent Agent => this.Mock.Object;
public int CallCount { get; private set; }
public List<List<ChatMessage>> MessagesPerCall { get; } = [];
}
@@ -0,0 +1,259 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Moq;
namespace Microsoft.Agents.AI.UnitTests;
/// <summary>
/// Unit tests for the <see cref="TodoCompletionLoopEvaluator"/> class.
/// </summary>
public class TodoCompletionLoopEvaluatorTests
{
/// <summary>
/// Verify that the constructor throws when a non-null but empty modes collection is supplied.
/// </summary>
[Fact]
public void TodoCompletionLoopEvaluator_EmptyModes_Throws()
{
// Act & Assert
Assert.Throws<ArgumentException>(() => new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = [] }));
}
/// <summary>
/// Verify that the constructor throws when a mode name is null, empty, or whitespace.
/// </summary>
/// <param name="mode">The invalid mode name.</param>
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public void TodoCompletionLoopEvaluator_InvalidModeName_Throws(string? mode)
{
// Act & Assert
Assert.Throws<ArgumentException>(() => new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = [mode!] }));
}
/// <summary>
/// Verify that the constructor succeeds with null modes (applies in every mode) and with a valid mode set.
/// </summary>
[Fact]
public void TodoCompletionLoopEvaluator_ValidConstruction_Succeeds()
{
// Act & Assert
_ = new TodoCompletionLoopEvaluator();
_ = new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = ["execute"] });
}
/// <summary>
/// Verify that evaluation throws when no <see cref="TodoProvider"/> can be resolved from the agent.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoTodoProvider_ThrowsAsync()
{
// Arrange — a bare agent that resolves no providers.
var evaluator = new TodoCompletionLoopEvaluator();
var context = CreateContext(new Mock<AIAgent>().Object, new ChatClientAgentSession());
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(async () => await evaluator.EvaluateAsync(context));
}
/// <summary>
/// Verify that evaluation throws when modes are configured but no <see cref="AgentModeProvider"/> can be resolved.
/// </summary>
[Fact]
public async Task EvaluateAsync_ModesConfiguredButNoModeProvider_ThrowsAsync()
{
// Arrange — agent has a TodoProvider but no AgentModeProvider.
var todoProvider = new TodoProvider();
var session = new ChatClientAgentSession();
SeedTodos(session, (1, "Task one", null, false));
AIAgent agent = CreateAgent(todoProvider);
var evaluator = new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = ["execute"] });
LoopContext context = CreateContext(agent, session);
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(async () => await evaluator.EvaluateAsync(context));
}
/// <summary>
/// Verify that, with no modes configured, the evaluator continues while incomplete todos remain and the feedback
/// lists those todos.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoModes_RemainingTodos_ContinuesWithFeedbackAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var session = new ChatClientAgentSession();
SeedTodos(session, (1, "Write code", null, false), (2, "Add tests", "cover edge cases", false), (3, "Done item", null, true));
AIAgent agent = CreateAgent(todoProvider);
var evaluator = new TodoCompletionLoopEvaluator();
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.NotNull(evaluation.Feedback);
Assert.Contains("Write code", evaluation.Feedback!);
Assert.Contains("Add tests", evaluation.Feedback!);
Assert.Contains("cover edge cases", evaluation.Feedback!);
// Completed items must not appear in the feedback list.
Assert.DoesNotContain("Done item", evaluation.Feedback!);
}
/// <summary>
/// Verify that, with no modes configured, the evaluator stops when there are no incomplete todos.
/// </summary>
[Fact]
public async Task EvaluateAsync_NoModes_NoRemainingTodos_StopsAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var session = new ChatClientAgentSession();
SeedTodos(session, (1, "Completed", null, true));
AIAgent agent = CreateAgent(todoProvider);
var evaluator = new TodoCompletionLoopEvaluator();
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that, when the current mode is one of the configured modes and incomplete todos remain, the evaluator
/// continues.
/// </summary>
[Fact]
public async Task EvaluateAsync_ModeMatches_RemainingTodos_ContinuesAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var modeProvider = new AgentModeProvider();
var session = new ChatClientAgentSession();
modeProvider.SetMode(session, "execute");
SeedTodos(session, (1, "Work item", null, false));
AIAgent agent = CreateAgent(todoProvider, modeProvider);
var evaluator = new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = ["execute"] });
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that, when the current mode is one of the configured modes but no incomplete todos remain, the evaluator
/// stops.
/// </summary>
[Fact]
public async Task EvaluateAsync_ModeMatches_NoRemainingTodos_StopsAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var modeProvider = new AgentModeProvider();
var session = new ChatClientAgentSession();
modeProvider.SetMode(session, "execute");
SeedTodos(session, (1, "Already done", null, true));
AIAgent agent = CreateAgent(todoProvider, modeProvider);
var evaluator = new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = ["execute"] });
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that, when the current mode is not one of the configured modes, the evaluator stops even if incomplete
/// todos remain.
/// </summary>
[Fact]
public async Task EvaluateAsync_ModeDoesNotMatch_StopsEvenWithRemainingTodosAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var modeProvider = new AgentModeProvider();
var session = new ChatClientAgentSession();
modeProvider.SetMode(session, "plan");
SeedTodos(session, (1, "Still open", null, false));
AIAgent agent = CreateAgent(todoProvider, modeProvider);
var evaluator = new TodoCompletionLoopEvaluator(new TodoCompletionLoopEvaluatorOptions { Modes = ["execute"] });
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.False(evaluation.ShouldReinvoke);
}
/// <summary>
/// Verify that a custom feedback template with the remaining-todos placeholder is honored.
/// </summary>
[Fact]
public async Task EvaluateAsync_CustomTemplate_IsHonoredAsync()
{
// Arrange
var todoProvider = new TodoProvider();
var session = new ChatClientAgentSession();
SeedTodos(session, (1, "Remaining task", null, false));
AIAgent agent = CreateAgent(todoProvider);
var options = new TodoCompletionLoopEvaluatorOptions
{
FeedbackMessageTemplate = "Keep going. Open:\n" + TodoCompletionLoopEvaluator.RemainingTodosPlaceholder,
};
var evaluator = new TodoCompletionLoopEvaluator(options: options);
LoopContext context = CreateContext(agent, session);
// Act
LoopEvaluation evaluation = await evaluator.EvaluateAsync(context);
// Assert
Assert.True(evaluation.ShouldReinvoke);
Assert.StartsWith("Keep going. Open:", evaluation.Feedback);
Assert.Contains("Remaining task", evaluation.Feedback!);
}
private static ChatClientAgent CreateAgent(params AIContextProvider[] providers)
{
var chatClient = new Mock<IChatClient>().Object;
return new ChatClientAgent(chatClient, new ChatClientAgentOptions { AIContextProviders = providers });
}
private static LoopContext CreateContext(AIAgent agent, AgentSession session) => new(
agent,
session,
[new ChatMessage(ChatRole.User, "do the work")],
new AgentResponse([new ChatMessage(ChatRole.Assistant, "in progress")]));
private static void SeedTodos(AgentSession session, params (int Id, string Title, string? Description, bool IsComplete)[] items)
{
var state = new TodoState { NextId = items.Length + 1 };
foreach ((int id, string title, string? description, bool isComplete) in items)
{
state.Items.Add(new TodoItem
{
Id = id,
Title = title,
Description = description,
IsComplete = isComplete,
});
}
// Persist under the TodoProvider's state key so the provider reads it back via GetRemainingTodosAsync.
session.StateBag.SetValue(nameof(TodoProvider), state, AgentJsonUtilities.DefaultOptions);
}
}