chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using Microsoft.Agents.AI.DurableTask.State;
|
||||
using Microsoft.DurableTask;
|
||||
using Microsoft.DurableTask.Client;
|
||||
using Microsoft.DurableTask.Client.Entities;
|
||||
using Microsoft.DurableTask.Entities;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Chat;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for scenarios where an external client interacts with Durable Task Agents.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class AgentEntityTests(ITestOutputHelper outputHelper) : IDisposable
|
||||
{
|
||||
private const string DisabledDueToFailingCiJob = "Disabled due to persistent CI failures. See #6732.";
|
||||
|
||||
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
|
||||
? TimeSpan.FromMinutes(5)
|
||||
: TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly IConfiguration s_configuration =
|
||||
new ConfigurationBuilder()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
private readonly ITestOutputHelper _outputHelper = outputHelper;
|
||||
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
|
||||
|
||||
private CancellationToken TestTimeoutToken => this._cts.Token;
|
||||
|
||||
public void Dispose() => this._cts.Dispose();
|
||||
|
||||
[Fact(Skip = DisabledDueToFailingCiJob)]
|
||||
public async Task EntityNamePrefixAsync()
|
||||
{
|
||||
// Setup
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "TestAgent",
|
||||
instructions: "You are a helpful assistant that always responds with a friendly greeting."
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper);
|
||||
|
||||
// A proxy agent is needed to call the hosted test agent
|
||||
AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
|
||||
AgentSession session = await simpleAgentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
|
||||
AgentSessionId sessionId = session.GetService<AgentSessionId>();
|
||||
EntityInstanceId expectedEntityId = new($"dafx-{simpleAgent.Name}", sessionId.Key);
|
||||
|
||||
EntityMetadata? entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken);
|
||||
|
||||
Assert.Null(entity);
|
||||
|
||||
// Act: send a prompt to the agent
|
||||
await simpleAgentProxy.RunAsync(
|
||||
message: "Hello!",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Assert: verify the agent state was stored with the correct entity name prefix
|
||||
entity = await client.Entities.GetEntityAsync(expectedEntityId, true, this.TestTimeoutToken);
|
||||
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
DurableAgentState state = entity.State.ReadAs<DurableAgentState>();
|
||||
|
||||
DurableAgentStateRequest request = Assert.Single(state.Data.ConversationHistory.OfType<DurableAgentStateRequest>());
|
||||
|
||||
Assert.Null(request.OrchestrationId);
|
||||
}
|
||||
|
||||
[Theory(Skip = DisabledDueToFailingCiJob)]
|
||||
[InlineData("run")]
|
||||
[InlineData("Run")]
|
||||
[InlineData("RunAgentAsync")]
|
||||
public async Task RunAgentMethodNamesAllWorkAsync(string runAgentMethodName)
|
||||
{
|
||||
// Setup
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "TestAgent",
|
||||
instructions: "You are a helpful assistant that always responds with a friendly greeting."
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper);
|
||||
|
||||
// A proxy agent is needed to call the hosted test agent
|
||||
AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
|
||||
AgentSession session = await simpleAgentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
|
||||
AgentSessionId sessionId = session.GetService<AgentSessionId>();
|
||||
EntityInstanceId expectedEntityId = new($"dafx-{simpleAgent.Name}", sessionId.Key);
|
||||
|
||||
EntityMetadata? entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken);
|
||||
|
||||
Assert.Null(entity);
|
||||
|
||||
// Act: send a prompt to the agent
|
||||
await client.Entities.SignalEntityAsync(
|
||||
expectedEntityId,
|
||||
runAgentMethodName,
|
||||
new RunRequest("Hello!"),
|
||||
cancellation: this.TestTimeoutToken);
|
||||
|
||||
while (!this.TestTimeoutToken.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(500, this.TestTimeoutToken);
|
||||
|
||||
// Assert: verify the agent state was stored with the correct entity name prefix
|
||||
entity = await client.Entities.GetEntityAsync(expectedEntityId, true, this.TestTimeoutToken);
|
||||
|
||||
if (entity is not null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
DurableAgentState state = entity.State.ReadAs<DurableAgentState>();
|
||||
|
||||
DurableAgentStateRequest request = Assert.Single(state.Data.ConversationHistory.OfType<DurableAgentStateRequest>());
|
||||
|
||||
Assert.Null(request.OrchestrationId);
|
||||
}
|
||||
|
||||
[Fact(Skip = DisabledDueToFailingCiJob)]
|
||||
public async Task OrchestrationIdSetDuringOrchestrationAsync()
|
||||
{
|
||||
// Arrange
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "TestAgent",
|
||||
instructions: "You are a helpful assistant that always responds with a friendly greeting."
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start(
|
||||
[simpleAgent],
|
||||
this._outputHelper,
|
||||
registry => registry.AddOrchestrator<TestOrchestrator>());
|
||||
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
|
||||
// Act
|
||||
string orchestrationId = await client.ScheduleNewOrchestrationInstanceAsync(nameof(TestOrchestrator), "What is the capital of Maine?");
|
||||
|
||||
OrchestrationMetadata? status = await client.WaitForInstanceCompletionAsync(
|
||||
orchestrationId,
|
||||
true,
|
||||
this.TestTimeoutToken);
|
||||
|
||||
// Assert
|
||||
EntityInstanceId expectedEntityId = AgentSessionId.Parse(status.ReadOutputAs<string>()!);
|
||||
|
||||
EntityMetadata? entity = await client.Entities.GetEntityAsync(expectedEntityId, true, this.TestTimeoutToken);
|
||||
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
DurableAgentState state = entity.State.ReadAs<DurableAgentState>();
|
||||
|
||||
DurableAgentStateRequest request = Assert.Single(state.Data.ConversationHistory.OfType<DurableAgentStateRequest>());
|
||||
|
||||
Assert.Equal(orchestrationId, request.OrchestrationId);
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Constructed via reflection.")]
|
||||
private sealed class TestOrchestrator : TaskOrchestrator<string, string>
|
||||
{
|
||||
public override async Task<string> RunAsync(TaskOrchestrationContext context, string input)
|
||||
{
|
||||
DurableAIAgent writer = context.GetAgent("TestAgent");
|
||||
AgentSession writerSession = await writer.CreateSessionAsync();
|
||||
|
||||
await writer.RunAsync(
|
||||
message: context.GetInput<string>()!,
|
||||
session: writerSession);
|
||||
|
||||
AgentSessionId sessionId = writerSession.GetService<AgentSessionId>();
|
||||
|
||||
return sessionId.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for validating the durable agent console app samples
|
||||
/// located in samples/Durable/Agents/ConsoleApps.
|
||||
/// </summary>
|
||||
[Collection("Samples")]
|
||||
[Trait("Category", "SampleValidation")]
|
||||
public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
|
||||
{
|
||||
private static readonly string s_samplesPath = Path.GetFullPath(
|
||||
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableAgents", "ConsoleApps"));
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override string SamplesPath => s_samplesPath;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override bool RequiresRedis => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void ConfigureAdditionalEnvironmentVariables(ProcessStartInfo startInfo, Action<string, string> setEnvVar)
|
||||
{
|
||||
setEnvVar("REDIS_CONNECTION_STRING", $"localhost:{RedisPort}");
|
||||
}
|
||||
|
||||
[Fact(Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task SingleAgentSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
||||
string samplePath = Path.Combine(s_samplesPath, "01_SingleAgent");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
string agentResponse = string.Empty;
|
||||
bool inputSent = false;
|
||||
|
||||
// Read output from logs queue
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
// Look for the agent's response. Unlike the interactive mode, we won't actually see a line
|
||||
// that starts with "Joker: ". Instead, we'll see a line that looks like "You: Joker: ..." because
|
||||
// the standard input is *not* echoed back to standard output.
|
||||
if (line.Contains("Joker: ", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// This will give us the first line of the agent's response, which is all we need to verify that the agent is working.
|
||||
agentResponse = line.Substring("Joker: ".Length).Trim();
|
||||
break;
|
||||
}
|
||||
else if (!inputSent)
|
||||
{
|
||||
// Send input to stdin after we've started seeing output from the app
|
||||
await this.WriteInputAsync(process, "Tell me a joke about a pirate.", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the agent");
|
||||
Assert.NotEmpty(agentResponse);
|
||||
|
||||
// Send exit command
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
||||
string samplePath = Path.Combine(s_samplesPath, "02_AgentOrchestration_Chaining");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
// Console app runs automatically, just wait for completion
|
||||
string? line;
|
||||
bool foundSuccess = false;
|
||||
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (line.Contains("Orchestration completed successfully!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundSuccess = true;
|
||||
}
|
||||
|
||||
if (line.Contains("Result:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
string result = line.Substring("Result:".Length).Trim();
|
||||
Assert.NotEmpty(result);
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for failure
|
||||
if (line.Contains("Orchestration failed!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail("Orchestration failed.");
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(foundSuccess, "Orchestration did not complete successfully.");
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task MultiAgentConcurrencySampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
||||
string samplePath = Path.Combine(s_samplesPath, "03_AgentOrchestration_Concurrency");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
// Send input to stdin
|
||||
await this.WriteInputAsync(process, "What is temperature?", testTimeoutCts.Token);
|
||||
|
||||
// Read output from logs queue
|
||||
StringBuilder output = new();
|
||||
string? line;
|
||||
bool foundSuccess = false;
|
||||
bool foundPhysicist = false;
|
||||
bool foundChemist = false;
|
||||
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
output.AppendLine(line);
|
||||
|
||||
if (line.Contains("Orchestration completed successfully!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundSuccess = true;
|
||||
}
|
||||
|
||||
if (line.Contains("Physicist's response:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundPhysicist = true;
|
||||
}
|
||||
|
||||
if (line.Contains("Chemist's response:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundChemist = true;
|
||||
}
|
||||
|
||||
// Check for failure
|
||||
if (line.Contains("Orchestration failed!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail("Orchestration failed.");
|
||||
}
|
||||
|
||||
// Stop reading once we have both responses
|
||||
if (foundSuccess && foundPhysicist && foundChemist)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(foundSuccess, "Orchestration did not complete successfully.");
|
||||
Assert.True(foundPhysicist, "Physicist response not found.");
|
||||
Assert.True(foundChemist, "Chemist response not found.");
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task MultiAgentConditionalSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
|
||||
string samplePath = Path.Combine(s_samplesPath, "04_AgentOrchestration_Conditionals");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
// Test with legitimate email
|
||||
await this.TestSpamDetectionAsync(
|
||||
process: process,
|
||||
logs: logs,
|
||||
emailId: "email-001",
|
||||
emailContent: "Hi John. I wanted to follow up on our meeting yesterday about the quarterly report. Could you please send me the updated figures by Friday? Thanks!",
|
||||
expectedSpam: false,
|
||||
testTimeoutCts.Token);
|
||||
|
||||
// Restart the process for the second test
|
||||
await process.WaitForExitAsync();
|
||||
});
|
||||
|
||||
// Run second test with spam email
|
||||
using CancellationTokenSource testTimeoutCts2 = this.CreateTestTimeoutCts();
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
await this.TestSpamDetectionAsync(
|
||||
process,
|
||||
logs,
|
||||
emailId: "email-002",
|
||||
emailContent: "URGENT! You've won $1,000,000! Click here now to claim your prize! Limited time offer! Don't miss out!",
|
||||
expectedSpam: true,
|
||||
testTimeoutCts2.Token);
|
||||
});
|
||||
}
|
||||
|
||||
private async Task TestSpamDetectionAsync(
|
||||
Process process,
|
||||
BlockingCollection<OutputLog> logs,
|
||||
string emailId,
|
||||
string emailContent,
|
||||
bool expectedSpam,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Send email content to stdin
|
||||
await this.WriteInputAsync(process, emailContent, cancellationToken);
|
||||
|
||||
// Read output from logs queue
|
||||
string? line;
|
||||
bool foundSuccess = false;
|
||||
|
||||
while ((line = this.ReadLogLine(logs, cancellationToken)) != null)
|
||||
{
|
||||
if (line.Contains("Email sent", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.False(expectedSpam, "Email was sent, but was expected to be marked as spam.");
|
||||
}
|
||||
|
||||
if (line.Contains("Email marked as spam", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.True(expectedSpam, "Email was marked as spam, but was expected to be sent.");
|
||||
}
|
||||
|
||||
if (line.Contains("Orchestration completed successfully!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundSuccess = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for failure
|
||||
if (line.Contains("Orchestration failed!", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail("Orchestration failed.");
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(foundSuccess, "Orchestration did not complete successfully.");
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures.")]
|
||||
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
|
||||
{
|
||||
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
|
||||
|
||||
// Start the HITL orchestration following the happy path from README
|
||||
await this.WriteInputAsync(process, "The Future of Artificial Intelligence", testTimeoutCts.Token);
|
||||
await this.WriteInputAsync(process, "3", testTimeoutCts.Token);
|
||||
await this.WriteInputAsync(process, "72", testTimeoutCts.Token);
|
||||
|
||||
// Read output from logs queue
|
||||
string? line;
|
||||
bool rejectionSent = false;
|
||||
bool approvalSent = false;
|
||||
bool contentPublished = false;
|
||||
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
// Look for notification that content is ready. The first time we see this, we should send a rejection.
|
||||
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
|
||||
if (line.Contains("Content is ready for review", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!rejectionSent)
|
||||
{
|
||||
// Prompt: Approve? (y/n):
|
||||
await this.WriteInputAsync(process, "n", testTimeoutCts.Token);
|
||||
|
||||
// Prompt: Feedback (optional):
|
||||
await this.WriteInputAsync(
|
||||
process,
|
||||
"The article needs more technical depth and better examples. Rewrite it with less than 300 words.",
|
||||
testTimeoutCts.Token);
|
||||
rejectionSent = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
|
||||
await this.WriteInputAsync(process, "y", testTimeoutCts.Token);
|
||||
|
||||
// Prompt: Feedback (optional):
|
||||
await this.WriteInputAsync(process, "Looks good!", testTimeoutCts.Token);
|
||||
approvalSent = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Look for success message
|
||||
if (line.Contains("PUBLISHING: Content has been published", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
contentPublished = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for failure
|
||||
if (line.Contains("Orchestration failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail("Orchestration failed.");
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(rejectionSent, "Wasn't prompted with the first draft.");
|
||||
Assert.True(approvalSent, "Wasn't prompted with the second draft.");
|
||||
Assert.True(contentPublished, "Content was not published.");
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures.")]
|
||||
public async Task LongRunningToolsSampleValidationAsync()
|
||||
{
|
||||
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
|
||||
|
||||
// Test starting an agent that schedules a content generation orchestration
|
||||
await this.WriteInputAsync(
|
||||
process,
|
||||
"Start a content generation workflow for the topic 'The Future of Artificial Intelligence'. Keep it less than 300 words.",
|
||||
testTimeoutCts.Token);
|
||||
|
||||
// Read output from logs queue
|
||||
bool rejectionSent = false;
|
||||
bool approvalSent = false;
|
||||
bool contentPublished = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
// Look for notification that content is ready. The first time we see this, we should send a rejection.
|
||||
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
|
||||
if (line.Contains("NOTIFICATION: Please review the following content for approval", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Wait for the notification to be fully written to the console
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), testTimeoutCts.Token);
|
||||
|
||||
if (!rejectionSent)
|
||||
{
|
||||
// Reject the content with feedback. Note that we need to send a newline character to the console first before sending the input.
|
||||
await this.WriteInputAsync(
|
||||
process,
|
||||
"\nReject the content with feedback: Make it even shorter.",
|
||||
testTimeoutCts.Token);
|
||||
rejectionSent = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
|
||||
await this.WriteInputAsync(
|
||||
process,
|
||||
"\nApprove the content",
|
||||
testTimeoutCts.Token);
|
||||
approvalSent = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Look for success message
|
||||
if (line.Contains("PUBLISHING: Content has been published successfully", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
contentPublished = true;
|
||||
|
||||
// Ask for the status of the workflow to confirm that it completed successfully.
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), testTimeoutCts.Token);
|
||||
await this.WriteInputAsync(process, "\nGet the status of the workflow you previously started", testTimeoutCts.Token);
|
||||
}
|
||||
|
||||
// Check for workflow completion or failure
|
||||
if (contentPublished)
|
||||
{
|
||||
if (line.Contains("Completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
break;
|
||||
}
|
||||
else if (line.Contains("Failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail("Workflow failed.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(rejectionSent, "Wasn't prompted with the first draft.");
|
||||
Assert.True(approvalSent, "Wasn't prompted with the second draft.");
|
||||
Assert.True(contentPublished, "Content was not published.");
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures.")]
|
||||
public async Task ReliableStreamingSampleValidationAsync()
|
||||
{
|
||||
string samplePath = Path.Combine(s_samplesPath, "07_ReliableStreaming");
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(150));
|
||||
|
||||
// Test the agent endpoint with a simple prompt
|
||||
await this.WriteInputAsync(process, "Plan a 5-day trip to Seattle. Include daily activities.", testTimeoutCts.Token);
|
||||
|
||||
// Read output from stdout - should stream in real-time
|
||||
// NOTE: The sample uses Console.Write() for streaming chunks, which means content may not be line-buffered.
|
||||
// We test the interrupt/resume flow by:
|
||||
// 1. Waiting for at least 10 lines of content
|
||||
// 2. Sending Enter to interrupt
|
||||
// 3. Verifying we get "Last cursor" output
|
||||
// 4. Sending Enter again to resume
|
||||
// 5. Verifying we get more content and that we're not restarting from the beginning
|
||||
string? line;
|
||||
bool foundConversationStart = false;
|
||||
int contentLinesBeforeInterrupt = 0;
|
||||
int contentLinesAfterResume = 0;
|
||||
bool foundLastCursor = false;
|
||||
bool foundResumeMessage = false;
|
||||
bool interrupted = false;
|
||||
bool resumed = false;
|
||||
|
||||
// Read output with a reasonable timeout
|
||||
using CancellationTokenSource readTimeoutCts = this.CreateTestTimeoutCts();
|
||||
DateTime? interruptTime = null;
|
||||
try
|
||||
{
|
||||
while ((line = this.ReadLogLine(logs, readTimeoutCts.Token)) != null)
|
||||
{
|
||||
// Look for the conversation start message (updated format)
|
||||
if (line.Contains("Conversation ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundConversationStart = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is a content line (not prompts or status messages)
|
||||
bool isContentLine = !string.IsNullOrWhiteSpace(line) &&
|
||||
!line.Contains("Conversation ID", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("Press [Enter]", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("You:", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("exit", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("Stream cancelled", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("Resuming conversation", StringComparison.OrdinalIgnoreCase) &&
|
||||
!line.Contains("Last cursor", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Phase 1: Collect content before interrupt
|
||||
if (foundConversationStart && !interrupted && isContentLine)
|
||||
{
|
||||
contentLinesBeforeInterrupt++;
|
||||
}
|
||||
|
||||
// Phase 2: Wait for enough content, then interrupt
|
||||
// Interrupt after 2 lines to maximize chance of catching stream while active
|
||||
// (streams can complete very quickly, so we need to interrupt early)
|
||||
if (foundConversationStart && !interrupted && contentLinesBeforeInterrupt >= 2)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Interrupting stream after {contentLinesBeforeInterrupt} content lines");
|
||||
interrupted = true;
|
||||
interruptTime = DateTime.Now;
|
||||
|
||||
// Send Enter to interrupt the stream
|
||||
await this.WriteInputAsync(process, string.Empty, testTimeoutCts.Token);
|
||||
|
||||
// Give the cancellation token a moment to be processed
|
||||
// Use a longer delay to ensure cancellation propagates
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300), testTimeoutCts.Token);
|
||||
}
|
||||
|
||||
// Phase 3: Look for "Last cursor" message after interrupt
|
||||
if (interrupted && !resumed && line.Contains("Last cursor", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundLastCursor = true;
|
||||
|
||||
// Send Enter again to resume
|
||||
this.OutputHelper.WriteLine("Resuming stream from last cursor");
|
||||
await this.WriteInputAsync(process, string.Empty, testTimeoutCts.Token);
|
||||
resumed = true;
|
||||
}
|
||||
|
||||
// Phase 4: Look for resume message
|
||||
if (resumed && line.Contains("Resuming conversation", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundResumeMessage = true;
|
||||
}
|
||||
|
||||
// Phase 5: Collect content after resume
|
||||
if (resumed && isContentLine)
|
||||
{
|
||||
contentLinesAfterResume++;
|
||||
}
|
||||
|
||||
// Look for completion message - but don't break if we interrupted and haven't found Last cursor yet
|
||||
// Allow some time after interrupt for the cancellation message to appear
|
||||
if (line.Contains("Conversation completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// If we interrupted but haven't found Last cursor, wait a bit more
|
||||
if (interrupted && !foundLastCursor && interruptTime.HasValue)
|
||||
{
|
||||
TimeSpan timeSinceInterrupt = DateTime.Now - interruptTime.Value;
|
||||
if (timeSinceInterrupt < TimeSpan.FromSeconds(2))
|
||||
{
|
||||
// Continue reading for a bit more to catch the cancellation message
|
||||
this.OutputHelper.WriteLine("Stream completed naturally, but waiting for Last cursor message after interrupt...");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Only break if we've completed the test or if stream completed without interruption
|
||||
if (!interrupted || (resumed && foundResumeMessage && contentLinesAfterResume >= 5))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Stop once we've verified the interrupt/resume flow works
|
||||
if (resumed && foundResumeMessage && contentLinesAfterResume >= 5)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Successfully verified interrupt/resume: {contentLinesBeforeInterrupt} lines before, {contentLinesAfterResume} lines after");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If we interrupted but didn't find Last cursor, wait a bit more for it to appear
|
||||
if (interrupted && !foundLastCursor && interruptTime.HasValue)
|
||||
{
|
||||
TimeSpan timeSinceInterrupt = DateTime.Now - interruptTime.Value;
|
||||
if (timeSinceInterrupt < TimeSpan.FromSeconds(3))
|
||||
{
|
||||
this.OutputHelper.WriteLine("Waiting for Last cursor message after interrupt...");
|
||||
using CancellationTokenSource waitCts = new(TimeSpan.FromSeconds(2));
|
||||
try
|
||||
{
|
||||
while ((line = this.ReadLogLine(logs, waitCts.Token)) != null)
|
||||
{
|
||||
if (line.Contains("Last cursor", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
foundLastCursor = true;
|
||||
if (!resumed)
|
||||
{
|
||||
this.OutputHelper.WriteLine("Resuming stream from last cursor");
|
||||
await this.WriteInputAsync(process, string.Empty, testTimeoutCts.Token);
|
||||
resumed = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Timeout waiting for Last cursor
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Timeout - check if we got enough to verify the flow
|
||||
this.OutputHelper.WriteLine($"Read timeout reached. Interrupted: {interrupted}, Resumed: {resumed}, Content before: {contentLinesBeforeInterrupt}, Content after: {contentLinesAfterResume}");
|
||||
}
|
||||
|
||||
Assert.True(foundConversationStart, "Conversation start message not found.");
|
||||
Assert.True(contentLinesBeforeInterrupt >= 2, $"Not enough content before interrupt (got {contentLinesBeforeInterrupt}).");
|
||||
|
||||
// If stream completed before interrupt could take effect, that's a timing issue
|
||||
// but we should still verify we got the conversation started
|
||||
if (!interrupted)
|
||||
{
|
||||
this.OutputHelper.WriteLine("WARNING: Stream completed before interrupt could be sent. This may indicate the stream is too fast.");
|
||||
}
|
||||
|
||||
Assert.True(interrupted, "Stream was not interrupted (may have completed too quickly).");
|
||||
Assert.True(foundLastCursor, "'Last cursor' message not found after interrupt.");
|
||||
Assert.True(resumed, "Stream was not resumed.");
|
||||
Assert.True(foundResumeMessage, "Resume message not found.");
|
||||
Assert.True(contentLinesAfterResume > 0, "No content received after resume (expected to continue from cursor, not restart).");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.ComponentModel;
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
|
||||
using Microsoft.DurableTask;
|
||||
using Microsoft.DurableTask.Client;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Chat;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for scenarios where an external client interacts with Durable Task Agents.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDisposable
|
||||
{
|
||||
private const string DisabledDueToFailingCiJob = "Disabled due to persistent CI failures. See #6732.";
|
||||
|
||||
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
|
||||
? TimeSpan.FromMinutes(5)
|
||||
: TimeSpan.FromSeconds(120);
|
||||
|
||||
private static readonly IConfiguration s_configuration =
|
||||
new ConfigurationBuilder()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
private readonly ITestOutputHelper _outputHelper = outputHelper;
|
||||
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
|
||||
|
||||
private CancellationToken TestTimeoutToken => this._cts.Token;
|
||||
|
||||
public void Dispose() => this._cts.Dispose();
|
||||
|
||||
[RetryFact(2, 5000, Skip = DisabledDueToFailingCiJob)]
|
||||
public async Task SimplePromptAsync()
|
||||
{
|
||||
// Setup
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
instructions: "You are a helpful assistant that always responds with a friendly greeting.",
|
||||
name: "TestAgent");
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper);
|
||||
|
||||
// A proxy agent is needed to call the hosted test agent
|
||||
AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
|
||||
// Act: send a prompt to the agent and wait for a response
|
||||
AgentSession session = await simpleAgentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
await simpleAgentProxy.RunAsync(
|
||||
message: "Hello!",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
AgentResponse response = await simpleAgentProxy.RunAsync(
|
||||
message: "Repeat what you just said but say it like a pirate",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Assert: verify the agent responded appropriately
|
||||
// We can't predict the exact response, but we can check that there is one response
|
||||
Assert.NotNull(response);
|
||||
Assert.NotEmpty(response.Text);
|
||||
|
||||
// Assert: verify the expected log entries were created in the expected category
|
||||
IReadOnlyCollection<LogEntry> logs = testHelper.GetLogs();
|
||||
Assert.NotEmpty(logs);
|
||||
List<LogEntry> agentLogs = [.. logs.Where(log => log.Category.Contains(simpleAgent.Name!)).ToList()];
|
||||
Assert.NotEmpty(agentLogs);
|
||||
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Hello!"));
|
||||
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = DisabledDueToFailingCiJob)]
|
||||
public async Task CallFunctionToolsAsync()
|
||||
{
|
||||
int weatherToolInvocationCount = 0;
|
||||
int packingListToolInvocationCount = 0;
|
||||
|
||||
string GetWeather(string location)
|
||||
{
|
||||
weatherToolInvocationCount++;
|
||||
return $"The weather in {location} is sunny with a high of 75°F and a low of 55°F.";
|
||||
}
|
||||
|
||||
string SuggestPackingList(string weather, bool isSunny)
|
||||
{
|
||||
packingListToolInvocationCount++;
|
||||
return isSunny ? "Pack sunglasses and sunscreen." : "Pack a raincoat and umbrella.";
|
||||
}
|
||||
|
||||
AIAgent tripPlanningAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
instructions: "You are a trip planning assistant. Use the weather tool and packing list tool as needed.",
|
||||
name: "TripPlanningAgent",
|
||||
description: "An agent to help plan your day trips",
|
||||
tools: [AIFunctionFactory.Create(GetWeather), AIFunctionFactory.Create(SuggestPackingList)]
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start([tripPlanningAgent], this._outputHelper);
|
||||
AIAgent tripPlanningAgentProxy = tripPlanningAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
|
||||
// Act: send a prompt to the agent
|
||||
AgentResponse response = await tripPlanningAgentProxy.RunAsync(
|
||||
message: "Help me figure out what to pack for my Seattle trip next Sunday",
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Assert: verify the agent responded appropriately
|
||||
// We can't predict the exact response, but we can check that there is one response
|
||||
Assert.NotNull(response);
|
||||
Assert.NotEmpty(response.Text);
|
||||
|
||||
// Assert: verify the expected log entries were created in the expected category
|
||||
IReadOnlyCollection<LogEntry> logs = testHelper.GetLogs();
|
||||
Assert.NotEmpty(logs);
|
||||
|
||||
List<LogEntry> agentLogs = [.. logs.Where(log => log.Category.Contains(tripPlanningAgent.Name!)).ToList()];
|
||||
Assert.NotEmpty(agentLogs);
|
||||
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Seattle trip"));
|
||||
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
|
||||
|
||||
// Assert: verify the tools were called
|
||||
Assert.Equal(1, weatherToolInvocationCount);
|
||||
Assert.Equal(1, packingListToolInvocationCount);
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = DisabledDueToFailingCiJob)]
|
||||
public async Task CallLongRunningFunctionToolsAsync()
|
||||
{
|
||||
[Description("Starts a greeting workflow and returns the workflow instance ID")]
|
||||
string StartWorkflowTool(string name)
|
||||
{
|
||||
return DurableAgentContext.Current.ScheduleNewOrchestration(nameof(RunWorkflowAsync), input: name);
|
||||
}
|
||||
|
||||
[Description("Gets the current status of a previously started workflow. A null response means the workflow has not started yet.")]
|
||||
static async Task<OrchestrationMetadata?> GetWorkflowStatusToolAsync(string instanceId)
|
||||
{
|
||||
OrchestrationMetadata? status = await DurableAgentContext.Current.GetOrchestrationStatusAsync(
|
||||
instanceId,
|
||||
includeDetails: true);
|
||||
if (status == null)
|
||||
{
|
||||
// If the status is not found, wait a bit before returning null to give the workflow time to start
|
||||
await Task.Delay(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
async Task<string> RunWorkflowAsync(TaskOrchestrationContext context, string name)
|
||||
{
|
||||
// 1. Get agent and create a session
|
||||
DurableAIAgent agent = context.GetAgent("SimpleAgent");
|
||||
AgentSession session = await agent.CreateSessionAsync(this.TestTimeoutToken);
|
||||
|
||||
// 2. Call an agent and tell it my name
|
||||
await agent.RunAsync($"My name is {name}.", session);
|
||||
|
||||
// 3. Call the agent again with the same session (ask it to tell me my name)
|
||||
AgentResponse response = await agent.RunAsync("What is my name?", session);
|
||||
|
||||
return response.Text;
|
||||
}
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start(
|
||||
this._outputHelper,
|
||||
configureAgents: agents =>
|
||||
{
|
||||
// This is the agent that will be used to start the workflow
|
||||
agents.AddAIAgentFactory(
|
||||
"WorkflowAgent",
|
||||
sp => TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "WorkflowAgent",
|
||||
instructions: "You can start greeting workflows and check their status.",
|
||||
services: sp,
|
||||
tools: [
|
||||
AIFunctionFactory.Create(StartWorkflowTool),
|
||||
AIFunctionFactory.Create(GetWorkflowStatusToolAsync)
|
||||
]));
|
||||
|
||||
// This is the agent that will be called by the workflow
|
||||
agents.AddAIAgent(TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "SimpleAgent",
|
||||
instructions: "You are a simple assistant."
|
||||
));
|
||||
},
|
||||
durableTaskRegistry: registry => registry.AddOrchestratorFunc<string, string>(nameof(RunWorkflowAsync), RunWorkflowAsync));
|
||||
|
||||
AIAgent workflowManagerAgentProxy = testHelper.Services.GetDurableAgentProxy("WorkflowAgent");
|
||||
|
||||
// Act: send a prompt to the agent
|
||||
AgentSession session = await workflowManagerAgentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
await workflowManagerAgentProxy.RunAsync(
|
||||
message: "Start a greeting workflow for \"John Doe\".",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Act: prompt it again to wait for the workflow to complete
|
||||
AgentResponse response = await workflowManagerAgentProxy.RunAsync(
|
||||
message: "Wait for the workflow to complete and tell me the result.",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Assert: verify the agent responded appropriately
|
||||
// We can't predict the exact response, but we can check that there is one response
|
||||
Assert.NotNull(response);
|
||||
Assert.NotEmpty(response.Text);
|
||||
Assert.Contains("John Doe", response.Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AsDurableAgentProxy_ThrowsWhenAgentNotRegistered()
|
||||
{
|
||||
// Setup: Register one agent but try to use a different one
|
||||
AIAgent registeredAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
instructions: "You are a helpful assistant.",
|
||||
name: "RegisteredAgent");
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start([registeredAgent], this._outputHelper);
|
||||
|
||||
// Create an agent with a different name that isn't registered
|
||||
AIAgent unregisteredAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
instructions: "You are a helpful assistant.",
|
||||
name: "UnregisteredAgent");
|
||||
|
||||
// Act & Assert: Should throw AgentNotRegisteredException
|
||||
AgentNotRegisteredException exception = Assert.Throws<AgentNotRegisteredException>(
|
||||
() => unregisteredAgent.AsDurableAgentProxy(testHelper.Services));
|
||||
|
||||
Assert.Equal("UnregisteredAgent", exception.AgentName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
|
||||
|
||||
internal sealed class LogEntry(
|
||||
string category,
|
||||
LogLevel level,
|
||||
EventId eventId,
|
||||
Exception? exception,
|
||||
string message,
|
||||
object? state,
|
||||
IReadOnlyList<KeyValuePair<string, object?>> contextProperties)
|
||||
{
|
||||
public string Category { get; } = category;
|
||||
|
||||
public DateTime Timestamp { get; } = DateTime.Now;
|
||||
|
||||
public EventId EventId { get; } = eventId;
|
||||
|
||||
public LogLevel LogLevel { get; } = level;
|
||||
|
||||
public Exception? Exception { get; } = exception;
|
||||
|
||||
public string Message { get; } = message;
|
||||
|
||||
public object? State { get; } = state;
|
||||
|
||||
public IReadOnlyList<KeyValuePair<string, object?>> ContextProperties { get; } = contextProperties;
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
string properties = this.ContextProperties.Count > 0
|
||||
? $"[{string.Join(", ", this.ContextProperties.Select(kvp => $"{kvp.Key}={kvp.Value}"))}] "
|
||||
: string.Empty;
|
||||
|
||||
string eventName = this.EventId.Name ?? string.Empty;
|
||||
string output = $"{this.Timestamp:o} [{this.Category}] {eventName} {properties}{this.Message}";
|
||||
|
||||
if (this.Exception is not null)
|
||||
{
|
||||
output += Environment.NewLine + this.Exception;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
|
||||
|
||||
internal sealed class TestLogger(string category, ITestOutputHelper output) : ILogger
|
||||
{
|
||||
private readonly string _category = category;
|
||||
private readonly ITestOutputHelper _output = output;
|
||||
private readonly ConcurrentQueue<LogEntry> _entries = new();
|
||||
|
||||
public IReadOnlyCollection<LogEntry> GetLogs() => this._entries;
|
||||
|
||||
public void ClearLogs() => this._entries.Clear();
|
||||
|
||||
IDisposable? ILogger.BeginScope<TState>(TState state) => null;
|
||||
|
||||
bool ILogger.IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
void ILogger.Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
LogEntry entry = new(
|
||||
category: this._category,
|
||||
level: logLevel,
|
||||
eventId: eventId,
|
||||
exception: exception,
|
||||
message: formatter(state, exception),
|
||||
state: state,
|
||||
contextProperties: []);
|
||||
|
||||
this._entries.Enqueue(entry);
|
||||
|
||||
try
|
||||
{
|
||||
this._output.WriteLine(entry.ToString());
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
// Expected when tests are shutting down
|
||||
}
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
|
||||
|
||||
internal sealed class TestLoggerProvider(ITestOutputHelper output) : ILoggerProvider
|
||||
{
|
||||
private readonly ITestOutputHelper _output = output ?? throw new ArgumentNullException(nameof(output));
|
||||
private readonly ConcurrentDictionary<string, TestLogger> _loggers = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public bool TryGetLogs(string category, out IReadOnlyCollection<LogEntry> logs)
|
||||
{
|
||||
if (this._loggers.TryGetValue(category, out TestLogger? logger))
|
||||
{
|
||||
logs = logger.GetLogs();
|
||||
return true;
|
||||
}
|
||||
|
||||
logs = [];
|
||||
return false;
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<LogEntry> GetAllLogs()
|
||||
{
|
||||
return this._loggers.Values
|
||||
.OfType<TestLogger>()
|
||||
.SelectMany(logger => logger.GetLogs())
|
||||
.ToList()
|
||||
.AsReadOnly();
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
foreach (TestLogger logger in this._loggers.Values.OfType<TestLogger>())
|
||||
{
|
||||
logger.ClearLogs();
|
||||
}
|
||||
}
|
||||
|
||||
ILogger ILoggerProvider.CreateLogger(string categoryName)
|
||||
{
|
||||
return this._loggers.GetOrAdd(categoryName, _ => new TestLogger(categoryName, this._output));
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>$(TargetFrameworksCore)</TargetFrameworks>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<InjectSharedIntegrationTestAzureCredentialsCode>True</InjectSharedIntegrationTestAzureCredentialsCode>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Public packages required by integration tests -->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.DurableTask.Client.AzureManaged" />
|
||||
<PackageReference Include="Microsoft.DurableTask.Worker.AzureManaged" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.DurableTask\Microsoft.Agents.AI.DurableTask.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,94 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using Microsoft.DurableTask;
|
||||
using Microsoft.DurableTask.Client;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Chat;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for orchestration execution scenarios with Durable Task Agents.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class OrchestrationTests(ITestOutputHelper outputHelper) : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
|
||||
? TimeSpan.FromMinutes(5)
|
||||
: TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly IConfiguration s_configuration =
|
||||
new ConfigurationBuilder()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
private readonly ITestOutputHelper _outputHelper = outputHelper;
|
||||
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
|
||||
|
||||
private CancellationToken TestTimeoutToken => this._cts.Token;
|
||||
|
||||
public void Dispose() => this._cts.Dispose();
|
||||
|
||||
[Fact]
|
||||
public async Task GetAgent_ThrowsWhenAgentNotRegisteredAsync()
|
||||
{
|
||||
// Define an orchestration that tries to use an unregistered agent
|
||||
static async Task<string> TestOrchestrationAsync(TaskOrchestrationContext context)
|
||||
{
|
||||
// Get an agent that hasn't been registered
|
||||
DurableAIAgent agent = context.GetAgent("NonExistentAgent");
|
||||
|
||||
// This should throw when RunAsync is called because the agent doesn't exist
|
||||
await agent.RunAsync("Hello");
|
||||
return "Should not reach here";
|
||||
}
|
||||
|
||||
// Setup: Create test helper without registering "NonExistentAgent"
|
||||
using TestHelper testHelper = TestHelper.Start(
|
||||
this._outputHelper,
|
||||
configureAgents: agents =>
|
||||
{
|
||||
// Register a different agent, but not "NonExistentAgent"
|
||||
agents.AddAIAgentFactory(
|
||||
"OtherAgent",
|
||||
sp => TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "OtherAgent",
|
||||
instructions: "You are a test agent."));
|
||||
},
|
||||
durableTaskRegistry: registry =>
|
||||
registry.AddOrchestratorFunc(
|
||||
name: nameof(TestOrchestrationAsync),
|
||||
orchestrator: TestOrchestrationAsync));
|
||||
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
|
||||
// Act: Start the orchestration
|
||||
string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(
|
||||
orchestratorName: nameof(TestOrchestrationAsync),
|
||||
cancellation: this.TestTimeoutToken);
|
||||
|
||||
// Wait for the orchestration to complete and check for failure
|
||||
OrchestrationMetadata status = await client.WaitForInstanceCompletionAsync(
|
||||
instanceId,
|
||||
getInputsAndOutputs: true,
|
||||
this.TestTimeoutToken);
|
||||
|
||||
// Assert: Verify the orchestration failed with the expected exception
|
||||
Assert.NotNull(status);
|
||||
Assert.Equal(OrchestrationRuntimeStatus.Failed, status.RuntimeStatus);
|
||||
Assert.NotNull(status.FailureDetails);
|
||||
|
||||
// Verify the exception type is AgentNotRegisteredException
|
||||
Assert.True(
|
||||
status.FailureDetails.ErrorType == typeof(AgentNotRegisteredException).FullName,
|
||||
$"Expected AgentNotRegisteredException but got ErrorType: {status.FailureDetails.ErrorType}, Message: {status.FailureDetails.ErrorMessage}");
|
||||
|
||||
// Verify the exception message contains the agent name
|
||||
Assert.Contains("NonExistentAgent", status.FailureDetails.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
+513
@@ -0,0 +1,513 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for sample validation integration tests providing shared infrastructure
|
||||
/// setup and utility methods for running console app samples.
|
||||
/// </summary>
|
||||
public abstract class SamplesValidationBase : IAsyncLifetime
|
||||
{
|
||||
protected const string DtsPort = "8080";
|
||||
protected const string RedisPort = "6379";
|
||||
|
||||
protected static readonly string DotnetTargetFramework = GetTargetFramework();
|
||||
protected static readonly IConfiguration Configuration =
|
||||
new ConfigurationBuilder()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
// Semaphores for thread-safe initialization of shared infrastructure.
|
||||
// xUnit may run tests in parallel, so we need to ensure that DTS emulator and Redis
|
||||
// are started only once across all test instances. Using SemaphoreSlim allows async-safe
|
||||
// locking, and the double-check pattern (check flag, acquire lock, check flag again)
|
||||
// minimizes lock contention after initialization is complete.
|
||||
private static readonly SemaphoreSlim s_dtsInitLock = new(1, 1);
|
||||
private static readonly SemaphoreSlim s_redisInitLock = new(1, 1);
|
||||
private static bool s_dtsInfrastructureStarted;
|
||||
private static bool s_redisInfrastructureStarted;
|
||||
|
||||
protected SamplesValidationBase(ITestOutputHelper outputHelper)
|
||||
{
|
||||
this.OutputHelper = outputHelper;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the test output helper for logging.
|
||||
/// </summary>
|
||||
protected ITestOutputHelper OutputHelper { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the base path to the samples directory for this test class.
|
||||
/// </summary>
|
||||
protected abstract string SamplesPath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether this test class requires Redis infrastructure.
|
||||
/// </summary>
|
||||
protected virtual bool RequiresRedis => false;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the task hub name prefix for this test class.
|
||||
/// </summary>
|
||||
protected virtual string TaskHubPrefix => "sample";
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
await EnsureDtsInfrastructureStartedAsync(this.OutputHelper, this.StartDtsEmulatorAsync);
|
||||
|
||||
if (this.RequiresRedis)
|
||||
{
|
||||
await EnsureRedisInfrastructureStartedAsync(this.OutputHelper, this.StartRedisAsync);
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures DTS infrastructure is started exactly once across all test instances.
|
||||
/// Static method writes to static field to avoid the code smell of instance methods modifying shared state.
|
||||
/// </summary>
|
||||
private static async Task EnsureDtsInfrastructureStartedAsync(ITestOutputHelper outputHelper, Func<Task> startAction)
|
||||
{
|
||||
if (s_dtsInfrastructureStarted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await s_dtsInitLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (!s_dtsInfrastructureStarted)
|
||||
{
|
||||
outputHelper.WriteLine("Starting shared DTS infrastructure...");
|
||||
await startAction();
|
||||
s_dtsInfrastructureStarted = true;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
s_dtsInitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures Redis infrastructure is started exactly once across all test instances.
|
||||
/// Static method writes to static field to avoid the code smell of instance methods modifying shared state.
|
||||
/// </summary>
|
||||
private static async Task EnsureRedisInfrastructureStartedAsync(ITestOutputHelper outputHelper, Func<Task> startAction)
|
||||
{
|
||||
if (s_redisInfrastructureStarted)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await s_redisInitLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (!s_redisInfrastructureStarted)
|
||||
{
|
||||
outputHelper.WriteLine("Starting shared Redis infrastructure...");
|
||||
await startAction();
|
||||
s_redisInfrastructureStarted = true;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
s_redisInitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
GC.SuppressFinalize(this);
|
||||
return default;
|
||||
}
|
||||
|
||||
protected sealed record OutputLog(DateTime Timestamp, LogLevel Level, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// Runs a sample test by starting the console app and executing the provided test action.
|
||||
/// </summary>
|
||||
protected async Task RunSampleTestAsync(string samplePath, Func<Process, BlockingCollection<OutputLog>, Task> testAction)
|
||||
{
|
||||
string uniqueTaskHubName = $"{this.TaskHubPrefix}-{Guid.NewGuid():N}"[..^26];
|
||||
|
||||
// Build the sample project first so that build failures are caught immediately
|
||||
// instead of silently failing inside 'dotnet run' and causing a timeout.
|
||||
await this.BuildSampleAsync(samplePath);
|
||||
|
||||
using BlockingCollection<OutputLog> logsContainer = [];
|
||||
using Process appProcess = this.StartConsoleApp(samplePath, logsContainer, uniqueTaskHubName);
|
||||
|
||||
try
|
||||
{
|
||||
await testAction(appProcess, logsContainer);
|
||||
}
|
||||
catch (OperationCanceledException e)
|
||||
{
|
||||
throw new TimeoutException("Core test logic timed out!", e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!logsContainer.IsAddingCompleted)
|
||||
{
|
||||
logsContainer.CompleteAdding();
|
||||
}
|
||||
|
||||
await this.StopProcessAsync(appProcess);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a line to the process's stdin and flushes it.
|
||||
/// </summary>
|
||||
protected async Task WriteInputAsync(Process process, string input, CancellationToken cancellationToken)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"{DateTime.Now:HH:mm:ss.fff} [{process.ProcessName}(in)]: {input}");
|
||||
await process.StandardInput.WriteLineAsync(input);
|
||||
await process.StandardInput.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the next Information-level log line from the queue.
|
||||
/// Returns null if cancelled or collection is completed.
|
||||
/// </summary>
|
||||
protected string? ReadLogLine(BlockingCollection<OutputLog> logs, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
OutputLog log = logs.Take(cancellationToken);
|
||||
|
||||
if (log.Message.Contains("Unhandled exception"))
|
||||
{
|
||||
Assert.Fail("Console app encountered an unhandled exception.");
|
||||
}
|
||||
|
||||
if (log.Level == LogLevel.Information)
|
||||
{
|
||||
return log.Message;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a cancellation token source with the specified timeout for test operations.
|
||||
/// </summary>
|
||||
protected CancellationTokenSource CreateTestTimeoutCts(TimeSpan? timeout = null)
|
||||
{
|
||||
TimeSpan testTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : timeout ?? TimeSpan.FromSeconds(120);
|
||||
return new CancellationTokenSource(testTimeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Allows derived classes to set additional environment variables for the console app process.
|
||||
/// </summary>
|
||||
protected virtual void ConfigureAdditionalEnvironmentVariables(ProcessStartInfo startInfo, Action<string, string> setEnvVar)
|
||||
{
|
||||
}
|
||||
|
||||
private static string GetTargetFramework()
|
||||
{
|
||||
string filePath = new Uri(typeof(SamplesValidationBase).Assembly.Location).LocalPath;
|
||||
string directory = Path.GetDirectoryName(filePath)!;
|
||||
string tfm = Path.GetFileName(directory);
|
||||
if (tfm.StartsWith("net", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return tfm;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Unable to find target framework in path: {filePath}");
|
||||
}
|
||||
|
||||
private async Task StartDtsEmulatorAsync()
|
||||
{
|
||||
if (!await this.IsDtsEmulatorRunningAsync())
|
||||
{
|
||||
this.OutputHelper.WriteLine("Starting DTS emulator...");
|
||||
await this.RunCommandAsync("docker", "run", "-d",
|
||||
"--name", "dts-emulator",
|
||||
"-p", $"{DtsPort}:8080",
|
||||
"-e", "DTS_USE_DYNAMIC_TASK_HUBS=true",
|
||||
"mcr.microsoft.com/dts/dts-emulator:latest");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task StartRedisAsync()
|
||||
{
|
||||
if (!await this.IsRedisRunningAsync())
|
||||
{
|
||||
this.OutputHelper.WriteLine("Starting Redis...");
|
||||
await this.RunCommandAsync("docker", "run", "-d",
|
||||
"--name", "redis",
|
||||
"-p", $"{RedisPort}:6379",
|
||||
"redis:latest");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> IsDtsEmulatorRunningAsync()
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Checking if DTS emulator is running at http://localhost:{DtsPort}/healthz...");
|
||||
|
||||
using HttpClient http2Client = new()
|
||||
{
|
||||
DefaultRequestVersion = new Version(2, 0),
|
||||
DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30));
|
||||
using HttpResponseMessage response = await http2Client.GetAsync(
|
||||
new Uri($"http://localhost:{DtsPort}/healthz"), timeoutCts.Token);
|
||||
|
||||
if (response.Content.Headers.ContentLength > 0)
|
||||
{
|
||||
string content = await response.Content.ReadAsStringAsync(timeoutCts.Token);
|
||||
this.OutputHelper.WriteLine($"DTS emulator health check response: {content}");
|
||||
}
|
||||
|
||||
bool isRunning = response.IsSuccessStatusCode;
|
||||
this.OutputHelper.WriteLine(isRunning ? "DTS emulator is running" : $"DTS emulator not running. Status: {response.StatusCode}");
|
||||
return isRunning;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"DTS emulator is not running: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> IsRedisRunningAsync()
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Checking if Redis is running at localhost:{RedisPort}...");
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30));
|
||||
ProcessStartInfo startInfo = new()
|
||||
{
|
||||
FileName = "docker",
|
||||
Arguments = "exec redis redis-cli ping",
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
using Process process = new() { StartInfo = startInfo };
|
||||
if (!process.Start())
|
||||
{
|
||||
this.OutputHelper.WriteLine("Failed to start docker exec command");
|
||||
return false;
|
||||
}
|
||||
|
||||
string output = await process.StandardOutput.ReadToEndAsync(timeoutCts.Token);
|
||||
await process.WaitForExitAsync(timeoutCts.Token);
|
||||
|
||||
bool isRunning = process.ExitCode == 0 && output.Contains("PONG", StringComparison.OrdinalIgnoreCase);
|
||||
this.OutputHelper.WriteLine(isRunning ? "Redis is running" : $"Redis not running. Exit: {process.ExitCode}, Output: {output}");
|
||||
return isRunning;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Redis is not running: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task BuildSampleAsync(string samplePath)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Building sample at {samplePath}...");
|
||||
|
||||
ProcessStartInfo buildInfo = new()
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = $"build --framework {DotnetTargetFramework}",
|
||||
WorkingDirectory = samplePath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
};
|
||||
|
||||
using Process buildProcess = new() { StartInfo = buildInfo };
|
||||
buildProcess.Start();
|
||||
|
||||
// Read both streams asynchronously to avoid deadlocks from filled pipe buffers
|
||||
Task<string> stdoutTask = buildProcess.StandardOutput.ReadToEndAsync();
|
||||
Task<string> stderrTask = buildProcess.StandardError.ReadToEndAsync();
|
||||
|
||||
using CancellationTokenSource buildCts = new(TimeSpan.FromMinutes(5));
|
||||
try
|
||||
{
|
||||
await buildProcess.WaitForExitAsync(buildCts.Token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
buildProcess.Kill(entireProcessTree: true);
|
||||
throw new TimeoutException($"Build timed out after 5 minutes for sample at {samplePath}.");
|
||||
}
|
||||
|
||||
await Task.WhenAll(stdoutTask, stderrTask);
|
||||
|
||||
string stdout = stdoutTask.Result;
|
||||
string stderr = stderrTask.Result;
|
||||
if (buildProcess.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to build sample at {samplePath}:\n{stdout}\n{stderr}");
|
||||
}
|
||||
|
||||
this.OutputHelper.WriteLine($"Build completed for {samplePath}.");
|
||||
}
|
||||
|
||||
private Process StartConsoleApp(string samplePath, BlockingCollection<OutputLog> logs, string taskHubName)
|
||||
{
|
||||
ProcessStartInfo startInfo = new()
|
||||
{
|
||||
FileName = "dotnet",
|
||||
Arguments = $"run --no-build --framework {DotnetTargetFramework}",
|
||||
WorkingDirectory = samplePath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardInput = true,
|
||||
};
|
||||
|
||||
string openAiEndpoint = Configuration["AZURE_OPENAI_ENDPOINT"] ??
|
||||
throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
|
||||
string openAiDeployment = Configuration["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] ??
|
||||
throw new InvalidOperationException("The required AZURE_OPENAI_CHAT_DEPLOYMENT_NAME env variable is not set.");
|
||||
|
||||
void SetAndLogEnvironmentVariable(string key, string value)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"Setting environment variable for {startInfo.FileName} sub-process: {key}={value}");
|
||||
startInfo.EnvironmentVariables[key] = value;
|
||||
}
|
||||
|
||||
SetAndLogEnvironmentVariable("AZURE_OPENAI_ENDPOINT", openAiEndpoint);
|
||||
SetAndLogEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT", openAiDeployment);
|
||||
SetAndLogEnvironmentVariable("DURABLE_TASK_SCHEDULER_CONNECTION_STRING",
|
||||
$"Endpoint=http://localhost:{DtsPort};TaskHub={taskHubName};Authentication=None");
|
||||
|
||||
this.ConfigureAdditionalEnvironmentVariables(startInfo, SetAndLogEnvironmentVariable);
|
||||
|
||||
Process process = new() { StartInfo = startInfo, EnableRaisingEvents = true };
|
||||
|
||||
process.ErrorDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "err", LogLevel.Error, logs);
|
||||
process.OutputDataReceived += (sender, e) => this.HandleProcessOutput(e.Data, startInfo.FileName, "out", LogLevel.Information, logs);
|
||||
|
||||
// When the process exits unexpectedly (e.g. build failure), complete the log collection
|
||||
// so that ReadLogLine returns null immediately instead of blocking until the test timeout.
|
||||
process.Exited += (sender, e) =>
|
||||
{
|
||||
if (!logs.IsAddingCompleted)
|
||||
{
|
||||
logs.CompleteAdding();
|
||||
}
|
||||
};
|
||||
|
||||
if (!process.Start())
|
||||
{
|
||||
throw new InvalidOperationException("Failed to start the console app");
|
||||
}
|
||||
|
||||
process.BeginErrorReadLine();
|
||||
process.BeginOutputReadLine();
|
||||
|
||||
return process;
|
||||
}
|
||||
|
||||
private void HandleProcessOutput(string? data, string processName, string stream, LogLevel level, BlockingCollection<OutputLog> logs)
|
||||
{
|
||||
if (data is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string logMessage = $"{DateTime.Now:HH:mm:ss.fff} [{processName}({stream})]: {data}";
|
||||
this.OutputHelper.WriteLine(logMessage);
|
||||
Debug.WriteLine(logMessage);
|
||||
|
||||
try
|
||||
{
|
||||
logs.Add(new OutputLog(DateTime.Now, level, data));
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
// Collection completed
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunCommandAsync(string command, params string[] args)
|
||||
{
|
||||
ProcessStartInfo startInfo = new()
|
||||
{
|
||||
FileName = command,
|
||||
Arguments = string.Join(" ", args),
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
|
||||
this.OutputHelper.WriteLine($"Running command: {command} {string.Join(" ", args)}");
|
||||
|
||||
using Process process = new() { StartInfo = startInfo };
|
||||
process.ErrorDataReceived += (sender, e) => this.OutputHelper.WriteLine($"[{command}(err)]: {e.Data}");
|
||||
process.OutputDataReceived += (sender, e) => this.OutputHelper.WriteLine($"[{command}(out)]: {e.Data}");
|
||||
|
||||
if (!process.Start())
|
||||
{
|
||||
throw new InvalidOperationException("Failed to start the command");
|
||||
}
|
||||
|
||||
process.BeginErrorReadLine();
|
||||
process.BeginOutputReadLine();
|
||||
|
||||
using CancellationTokenSource cts = new(TimeSpan.FromMinutes(1));
|
||||
await process.WaitForExitAsync(cts.Token);
|
||||
|
||||
this.OutputHelper.WriteLine($"Command completed with exit code: {process.ExitCode}");
|
||||
}
|
||||
|
||||
private async Task StopProcessAsync(Process process)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"{DateTime.Now:HH:mm:ss.fff} Killing process {process.ProcessName}#{process.Id}");
|
||||
process.Kill(entireProcessTree: true);
|
||||
|
||||
using CancellationTokenSource cts = new(TimeSpan.FromSeconds(10));
|
||||
await process.WaitForExitAsync(cts.Token);
|
||||
this.OutputHelper.WriteLine($"{DateTime.Now:HH:mm:ss.fff} Process exited: {process.Id}");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.OutputHelper.WriteLine($"{DateTime.Now:HH:mm:ss.fff} Failed to stop process: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure;
|
||||
using Azure.AI.OpenAI;
|
||||
using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging;
|
||||
using Microsoft.DurableTask;
|
||||
using Microsoft.DurableTask.Client;
|
||||
using Microsoft.DurableTask.Client.AzureManaged;
|
||||
using Microsoft.DurableTask.Worker;
|
||||
using Microsoft.DurableTask.Worker.AzureManaged;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenAI.Chat;
|
||||
using Shared.IntegrationTests;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
internal sealed class TestHelper : IDisposable
|
||||
{
|
||||
private readonly TestLoggerProvider _loggerProvider;
|
||||
private readonly IHost _host;
|
||||
private readonly DurableTaskClient _client;
|
||||
|
||||
// The static Start method should be used to create instances of this class.
|
||||
private TestHelper(
|
||||
TestLoggerProvider loggerProvider,
|
||||
IHost host,
|
||||
DurableTaskClient client)
|
||||
{
|
||||
this._loggerProvider = loggerProvider;
|
||||
this._host = host;
|
||||
this._client = client;
|
||||
}
|
||||
|
||||
public IServiceProvider Services => this._host.Services;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
this._host.Dispose();
|
||||
}
|
||||
|
||||
public bool TryGetLogs(string category, out IReadOnlyCollection<LogEntry> logs)
|
||||
=> this._loggerProvider.TryGetLogs(category, out logs);
|
||||
|
||||
public static TestHelper Start(
|
||||
AIAgent[] agents,
|
||||
ITestOutputHelper outputHelper,
|
||||
Action<DurableTaskRegistry>? durableTaskRegistry = null)
|
||||
{
|
||||
return BuildAndStartTestHelper(
|
||||
outputHelper,
|
||||
options => options.AddAIAgents(agents),
|
||||
durableTaskRegistry);
|
||||
}
|
||||
|
||||
public static TestHelper Start(
|
||||
ITestOutputHelper outputHelper,
|
||||
Action<DurableAgentsOptions> configureAgents,
|
||||
Action<DurableTaskRegistry>? durableTaskRegistry = null)
|
||||
{
|
||||
return BuildAndStartTestHelper(
|
||||
outputHelper,
|
||||
configureAgents,
|
||||
durableTaskRegistry);
|
||||
}
|
||||
|
||||
public DurableTaskClient GetClient() => this._client;
|
||||
|
||||
private static TestHelper BuildAndStartTestHelper(
|
||||
ITestOutputHelper outputHelper,
|
||||
Action<DurableAgentsOptions> configureAgents,
|
||||
Action<DurableTaskRegistry>? durableTaskRegistry)
|
||||
{
|
||||
TestLoggerProvider loggerProvider = new(outputHelper);
|
||||
|
||||
// Generate a unique TaskHub name for this test instance to prevent cross-test interference
|
||||
// when multiple tests run together and share the same DTS emulator.
|
||||
string uniqueTaskHubName = $"test-{Guid.NewGuid().ToString("N").Substring(0, 6)}";
|
||||
|
||||
IHost host = Host.CreateDefaultBuilder()
|
||||
.ConfigureServices((ctx, services) =>
|
||||
{
|
||||
string dtsConnectionString = GetDurableTaskSchedulerConnectionString(ctx.Configuration, uniqueTaskHubName);
|
||||
|
||||
// Register durable agents using the caller-supplied registration action and
|
||||
// apply the default chat client for agents that don't supply one themselves.
|
||||
services.ConfigureDurableAgents(
|
||||
options => configureAgents(options),
|
||||
workerBuilder: builder =>
|
||||
{
|
||||
builder.UseDurableTaskScheduler(dtsConnectionString);
|
||||
if (durableTaskRegistry != null)
|
||||
{
|
||||
builder.AddTasks(durableTaskRegistry);
|
||||
}
|
||||
},
|
||||
clientBuilder: builder => builder.UseDurableTaskScheduler(dtsConnectionString));
|
||||
})
|
||||
.ConfigureLogging((_, logging) =>
|
||||
{
|
||||
logging.AddProvider(loggerProvider);
|
||||
logging.SetMinimumLevel(LogLevel.Debug);
|
||||
})
|
||||
.Build();
|
||||
host.Start();
|
||||
|
||||
DurableTaskClient client = host.Services.GetRequiredService<DurableTaskClient>();
|
||||
return new TestHelper(loggerProvider, host, client);
|
||||
}
|
||||
|
||||
private static string GetDurableTaskSchedulerConnectionString(IConfiguration configuration, string? taskHubName = null)
|
||||
{
|
||||
// The default value is for local development using the Durable Task Scheduler emulator.
|
||||
string? connectionString = configuration["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"];
|
||||
|
||||
if (connectionString != null)
|
||||
{
|
||||
// If a connection string is provided, replace the TaskHub name if a custom one is specified
|
||||
if (taskHubName != null)
|
||||
{
|
||||
// Replace TaskHub in the connection string
|
||||
if (connectionString.Contains("TaskHub=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Find and replace the TaskHub value
|
||||
int taskHubIndex = connectionString.IndexOf("TaskHub=", StringComparison.OrdinalIgnoreCase);
|
||||
int taskHubValueStart = taskHubIndex + "TaskHub=".Length;
|
||||
int taskHubValueEnd = connectionString.IndexOf(';', taskHubValueStart);
|
||||
if (taskHubValueEnd == -1)
|
||||
{
|
||||
taskHubValueEnd = connectionString.Length;
|
||||
}
|
||||
|
||||
connectionString = string.Concat(
|
||||
connectionString.AsSpan(0, taskHubValueStart),
|
||||
taskHubName,
|
||||
connectionString.AsSpan(taskHubValueEnd));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Append TaskHub if it doesn't exist
|
||||
connectionString += $";TaskHub={taskHubName}";
|
||||
}
|
||||
}
|
||||
|
||||
return connectionString;
|
||||
}
|
||||
|
||||
// Default connection string with unique TaskHub name
|
||||
string defaultTaskHub = taskHubName ?? "default";
|
||||
return $"Endpoint=http://localhost:8080;TaskHub={defaultTaskHub};Authentication=None";
|
||||
}
|
||||
|
||||
internal static ChatClient GetAzureOpenAIChatClient(IConfiguration configuration)
|
||||
{
|
||||
string azureOpenAiEndpoint = configuration["AZURE_OPENAI_ENDPOINT"] ??
|
||||
throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set.");
|
||||
string azureOpenAiDeploymentName = configuration["AZURE_OPENAI_DEPLOYMENT_NAME"] ??
|
||||
throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT_NAME env variable is not set.");
|
||||
|
||||
// Check if AZURE_OPENAI_API_KEY is provided for key-based authentication.
|
||||
// NOTE: This is not used for automated tests, but can be useful for local development.
|
||||
string? azureOpenAiKey = configuration["AZURE_OPENAI_API_KEY"];
|
||||
|
||||
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
|
||||
? new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureKeyCredential(azureOpenAiKey))
|
||||
: new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), TestAzureCliCredentials.CreateAzureCliCredential());
|
||||
|
||||
return client.GetChatClient(azureOpenAiDeploymentName);
|
||||
}
|
||||
|
||||
internal IReadOnlyCollection<LogEntry> GetLogs()
|
||||
{
|
||||
return this._loggerProvider.GetAllLogs();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using Microsoft.Agents.AI.DurableTask.State;
|
||||
using Microsoft.DurableTask.Client;
|
||||
using Microsoft.DurableTask.Client.Entities;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Chat;
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for Time-To-Live (TTL) functionality of durable agent entities.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
[Trait("Category", "IntegrationDisabled")]
|
||||
public sealed class TimeToLiveTests(ITestOutputHelper outputHelper) : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
|
||||
? TimeSpan.FromMinutes(5)
|
||||
: TimeSpan.FromSeconds(30);
|
||||
|
||||
private static readonly IConfiguration s_configuration =
|
||||
new ConfigurationBuilder()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
private readonly ITestOutputHelper _outputHelper = outputHelper;
|
||||
private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout);
|
||||
|
||||
private CancellationToken TestTimeoutToken => this._cts.Token;
|
||||
|
||||
public void Dispose() => this._cts.Dispose();
|
||||
|
||||
[Fact]
|
||||
public async Task EntityExpiresAfterTTLAsync()
|
||||
{
|
||||
// Arrange: Create agent with short TTL (10 seconds)
|
||||
TimeSpan ttl = TimeSpan.FromSeconds(10);
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "TTLTestAgent",
|
||||
instructions: "You are a helpful assistant."
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start(
|
||||
this._outputHelper,
|
||||
options =>
|
||||
{
|
||||
options.DefaultTimeToLive = ttl;
|
||||
options.MinimumTimeToLiveSignalDelay = TimeSpan.FromSeconds(1);
|
||||
options.AddAIAgent(simpleAgent);
|
||||
});
|
||||
|
||||
AIAgent agentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
AgentSession session = await agentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
AgentSessionId sessionId = session.GetService<AgentSessionId>();
|
||||
|
||||
// Act: Send a message to the agent
|
||||
await agentProxy.RunAsync(
|
||||
message: "Hello!",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Verify entity exists and get expiration time
|
||||
EntityMetadata? entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
DurableAgentState state = entity.State.ReadAs<DurableAgentState>();
|
||||
Assert.NotNull(state.Data.ExpirationTimeUtc);
|
||||
DateTime expirationTime = state.Data.ExpirationTimeUtc.Value;
|
||||
Assert.True(expirationTime > DateTime.UtcNow);
|
||||
|
||||
// Calculate how long to wait: expiration time + buffer for signal processing
|
||||
TimeSpan waitTime = expirationTime - DateTime.UtcNow + TimeSpan.FromSeconds(1);
|
||||
if (waitTime > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(waitTime, this.TestTimeoutToken);
|
||||
}
|
||||
|
||||
// Poll the entity state until it's deleted (with timeout)
|
||||
DateTime pollTimeout = DateTime.UtcNow.AddSeconds(10);
|
||||
bool entityDeleted = false;
|
||||
while (DateTime.UtcNow < pollTimeout && !entityDeleted)
|
||||
{
|
||||
entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
entityDeleted = entity is null;
|
||||
|
||||
if (!entityDeleted)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), this.TestTimeoutToken);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert: Verify entity state is deleted
|
||||
Assert.True(entityDeleted, "Entity should have been deleted after TTL expiration");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EntityTTLResetsOnInteractionAsync()
|
||||
{
|
||||
// Arrange: Create agent with short TTL
|
||||
TimeSpan ttl = TimeSpan.FromSeconds(6);
|
||||
AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).AsAIAgent(
|
||||
name: "TTLResetTestAgent",
|
||||
instructions: "You are a helpful assistant."
|
||||
);
|
||||
|
||||
using TestHelper testHelper = TestHelper.Start(
|
||||
this._outputHelper,
|
||||
options =>
|
||||
{
|
||||
options.DefaultTimeToLive = ttl;
|
||||
options.MinimumTimeToLiveSignalDelay = TimeSpan.FromSeconds(1);
|
||||
options.AddAIAgent(simpleAgent);
|
||||
});
|
||||
|
||||
AIAgent agentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services);
|
||||
AgentSession session = await agentProxy.CreateSessionAsync(this.TestTimeoutToken);
|
||||
DurableTaskClient client = testHelper.GetClient();
|
||||
AgentSessionId sessionId = session.GetService<AgentSessionId>();
|
||||
|
||||
// Act: Send first message
|
||||
await agentProxy.RunAsync(
|
||||
message: "Hello!",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
EntityMetadata? entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
DurableAgentState state = entity.State.ReadAs<DurableAgentState>();
|
||||
DateTime firstExpirationTime = state.Data.ExpirationTimeUtc!.Value;
|
||||
|
||||
// Wait partway through TTL
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), this.TestTimeoutToken);
|
||||
|
||||
// Send second message (should reset TTL)
|
||||
await agentProxy.RunAsync(
|
||||
message: "Hello again!",
|
||||
session,
|
||||
cancellationToken: this.TestTimeoutToken);
|
||||
|
||||
// Verify expiration time was updated
|
||||
entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
state = entity.State.ReadAs<DurableAgentState>();
|
||||
DateTime secondExpirationTime = state.Data.ExpirationTimeUtc!.Value;
|
||||
Assert.True(secondExpirationTime > firstExpirationTime);
|
||||
|
||||
// Calculate when the original expiration time would have been
|
||||
DateTime originalExpirationTime = firstExpirationTime;
|
||||
TimeSpan waitUntilOriginalExpiration = originalExpirationTime - DateTime.UtcNow + TimeSpan.FromSeconds(2);
|
||||
|
||||
if (waitUntilOriginalExpiration > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(waitUntilOriginalExpiration, this.TestTimeoutToken);
|
||||
}
|
||||
|
||||
// Assert: Entity should still exist because TTL was reset
|
||||
// The new expiration time should be in the future
|
||||
entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
Assert.NotNull(entity);
|
||||
Assert.True(entity.IncludesState);
|
||||
|
||||
state = entity.State.ReadAs<DurableAgentState>();
|
||||
Assert.NotNull(state);
|
||||
Assert.NotNull(state.Data.ExpirationTimeUtc);
|
||||
Assert.True(
|
||||
state.Data.ExpirationTimeUtc > DateTime.UtcNow,
|
||||
"Entity should still be valid because TTL was reset");
|
||||
|
||||
// Wait for the entity to be deleted
|
||||
DateTime pollTimeout = DateTime.UtcNow.AddSeconds(10);
|
||||
bool entityDeleted = false;
|
||||
while (DateTime.UtcNow < pollTimeout && !entityDeleted)
|
||||
{
|
||||
entity = await client.Entities.GetEntityAsync(sessionId, true, this.TestTimeoutToken);
|
||||
entityDeleted = entity is null;
|
||||
|
||||
if (!entityDeleted)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), this.TestTimeoutToken);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert: Entity should have been deleted
|
||||
Assert.True(entityDeleted, "Entity should have been deleted after TTL expiration");
|
||||
}
|
||||
}
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for validating the durable workflow console app samples
|
||||
/// located in samples/04-hosting/DurableWorkflows/ConsoleApps.
|
||||
/// </summary>
|
||||
[Collection("Samples")]
|
||||
[Trait("Category", "SampleValidation")]
|
||||
public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
|
||||
{
|
||||
// In CI, `dotnet run` builds samples from scratch and LLM calls add latency, so 60s is not enough.
|
||||
private static readonly TimeSpan s_testTimeout = TimeSpan.FromSeconds(180);
|
||||
|
||||
private static readonly string s_samplesPath = Path.GetFullPath(
|
||||
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableWorkflows", "ConsoleApps"));
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override string SamplesPath => s_samplesPath;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override string TaskHubPrefix => "workflow";
|
||||
|
||||
[RetryFact(2, 5000)]
|
||||
public async Task SequentialWorkflowSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "01_SequentialWorkflow");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool inputSent = false;
|
||||
bool workflowCompleted = false;
|
||||
bool foundOrderLookup = false;
|
||||
bool foundOrderCancel = false;
|
||||
bool foundSendEmail = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
|
||||
if (inputSent)
|
||||
{
|
||||
foundOrderLookup |= line.Contains("[Activity] OrderLookup:", StringComparison.Ordinal);
|
||||
foundOrderCancel |= line.Contains("[Activity] OrderCancel:", StringComparison.Ordinal);
|
||||
foundSendEmail |= line.Contains("[Activity] SendEmail:", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("Workflow completed. Cancellation email sent for order 12345", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
workflowCompleted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the workflow.");
|
||||
Assert.True(foundOrderLookup, "OrderLookup executor log entry not found.");
|
||||
Assert.True(foundOrderCancel, "OrderCancel executor log entry not found.");
|
||||
Assert.True(foundSendEmail, "SendEmail executor log entry not found.");
|
||||
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task ConcurrentWorkflowSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "02_ConcurrentWorkflow");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool inputSent = false;
|
||||
bool workflowCompleted = false;
|
||||
bool foundParseQuestion = false;
|
||||
bool foundAggregator = false;
|
||||
bool foundAggregatorReceived2Responses = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (!inputSent && line.Contains("Enter a science question", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "What is gravity?", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
|
||||
if (inputSent)
|
||||
{
|
||||
foundParseQuestion |= line.Contains("[ParseQuestion]", StringComparison.Ordinal);
|
||||
foundAggregator |= line.Contains("[Aggregator]", StringComparison.Ordinal);
|
||||
foundAggregatorReceived2Responses |= line.Contains("Received 2 AI agent responses", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("Aggregation complete", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
workflowCompleted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the workflow.");
|
||||
Assert.True(foundParseQuestion, "ParseQuestion executor log entry not found.");
|
||||
Assert.True(foundAggregator, "Aggregator executor log entry not found.");
|
||||
Assert.True(foundAggregatorReceived2Responses, "Aggregator did not receive 2 AI agent responses.");
|
||||
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000)]
|
||||
public async Task ConditionalEdgesWorkflowSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "03_ConditionalEdges");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool validOrderSent = false;
|
||||
bool blockedOrderSent = false;
|
||||
bool validOrderCompleted = false;
|
||||
bool blockedOrderCompleted = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
// Send a valid order first (no 'B' in ID)
|
||||
if (!validOrderSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
||||
validOrderSent = true;
|
||||
}
|
||||
|
||||
// Check valid order completed (routed to PaymentProcessor)
|
||||
if (validOrderSent && !validOrderCompleted &&
|
||||
line.Contains("PaymentReferenceNumber", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
validOrderCompleted = true;
|
||||
|
||||
// Send a blocked order (contains 'B')
|
||||
await this.WriteInputAsync(process, "ORDER-B-999", testTimeoutCts.Token);
|
||||
blockedOrderSent = true;
|
||||
}
|
||||
|
||||
// Check blocked order completed (routed to NotifyFraud)
|
||||
if (blockedOrderSent && line.Contains("flagged as fraudulent", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
blockedOrderCompleted = true;
|
||||
break;
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(validOrderSent, "Valid order input was not sent.");
|
||||
Assert.True(validOrderCompleted, "Valid order did not complete (PaymentProcessor path).");
|
||||
Assert.True(blockedOrderSent, "Blocked order input was not sent.");
|
||||
Assert.True(blockedOrderCompleted, "Blocked order did not complete (NotifyFraud path).");
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
private void AssertNoError(string line)
|
||||
{
|
||||
if (line.Contains("Failed:", StringComparison.OrdinalIgnoreCase) ||
|
||||
line.Contains("Error:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Assert.Fail($"Workflow failed: {line}");
|
||||
}
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
||||
public async Task WorkflowEventsSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "05_WorkflowEvents");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool inputSent = false;
|
||||
bool foundStartedRun = false;
|
||||
bool foundExecutorInvoked = false;
|
||||
bool foundExecutorCompleted = false;
|
||||
bool foundLookupStarted = false;
|
||||
bool foundOrderFound = false;
|
||||
bool foundCancelProgress = false;
|
||||
bool foundOrderCancelled = false;
|
||||
bool foundEmailSent = false;
|
||||
bool foundYieldedOutput = false;
|
||||
bool foundWorkflowCompleted = false;
|
||||
bool foundCompletionResult = false;
|
||||
List<string> eventLines = [];
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (!inputSent && line.Contains("Enter order ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "12345", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
|
||||
if (inputSent)
|
||||
{
|
||||
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
||||
foundExecutorInvoked |= line.Contains("ExecutorInvokedEvent", StringComparison.Ordinal);
|
||||
foundExecutorCompleted |= line.Contains("ExecutorCompletedEvent", StringComparison.Ordinal);
|
||||
foundLookupStarted |= line.Contains("[Lookup] Looking up order", StringComparison.Ordinal);
|
||||
foundOrderFound |= line.Contains("[Lookup] Found:", StringComparison.Ordinal);
|
||||
foundCancelProgress |= line.Contains("[Cancel]", StringComparison.Ordinal) && line.Contains('%');
|
||||
foundOrderCancelled |= line.Contains("[Cancel] Done", StringComparison.Ordinal);
|
||||
foundEmailSent |= line.Contains("[Email] Sent to", StringComparison.Ordinal);
|
||||
foundYieldedOutput |= line.Contains("[Output]", StringComparison.Ordinal);
|
||||
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("Completed:", StringComparison.Ordinal))
|
||||
{
|
||||
foundCompletionResult = line.Contains("12345", StringComparison.Ordinal);
|
||||
break;
|
||||
}
|
||||
|
||||
// Collect event lines for ordering verification
|
||||
if (line.Contains("[Lookup]", StringComparison.Ordinal)
|
||||
|| line.Contains("[Cancel]", StringComparison.Ordinal)
|
||||
|| line.Contains("[Email]", StringComparison.Ordinal)
|
||||
|| line.Contains("[Output]", StringComparison.Ordinal))
|
||||
{
|
||||
eventLines.Add(line);
|
||||
}
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the workflow.");
|
||||
Assert.True(foundStartedRun, "Streaming run was not started.");
|
||||
Assert.True(foundExecutorInvoked, "ExecutorInvokedEvent not found in stream.");
|
||||
Assert.True(foundExecutorCompleted, "ExecutorCompletedEvent not found in stream.");
|
||||
Assert.True(foundLookupStarted, "OrderLookupStartedEvent not found in stream.");
|
||||
Assert.True(foundOrderFound, "OrderFoundEvent not found in stream.");
|
||||
Assert.True(foundCancelProgress, "CancellationProgressEvent not found in stream.");
|
||||
Assert.True(foundOrderCancelled, "OrderCancelledEvent not found in stream.");
|
||||
Assert.True(foundEmailSent, "EmailSentEvent not found in stream.");
|
||||
Assert.True(foundYieldedOutput, "WorkflowOutputEvent not found in stream.");
|
||||
Assert.True(foundWorkflowCompleted, "DurableWorkflowCompletedEvent not found in stream.");
|
||||
Assert.True(foundCompletionResult, "Completion result does not contain the order ID.");
|
||||
|
||||
// Verify event ordering: lookup events appear before cancel events, which appear before email events
|
||||
int lastLookupIndex = eventLines.FindLastIndex(l => l.Contains("[Lookup]", StringComparison.Ordinal));
|
||||
int firstCancelIndex = eventLines.FindIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
||||
int lastCancelIndex = eventLines.FindLastIndex(l => l.Contains("[Cancel]", StringComparison.Ordinal));
|
||||
int firstEmailIndex = eventLines.FindIndex(l => l.Contains("[Email]", StringComparison.Ordinal));
|
||||
|
||||
if (lastLookupIndex >= 0 && firstCancelIndex >= 0)
|
||||
{
|
||||
Assert.True(lastLookupIndex < firstCancelIndex, "Lookup events should appear before cancel events.");
|
||||
}
|
||||
|
||||
if (lastCancelIndex >= 0 && firstEmailIndex >= 0)
|
||||
{
|
||||
Assert.True(lastCancelIndex < firstEmailIndex, "Cancel events should appear before email events.");
|
||||
}
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
||||
public async Task WorkflowSharedStateSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "06_WorkflowSharedState");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool inputSent = false;
|
||||
bool foundStartedRun = false;
|
||||
bool foundValidateOutput = false;
|
||||
bool foundEnrichOutput = false;
|
||||
bool foundPaymentOutput = false;
|
||||
bool foundInvoiceOutput = false;
|
||||
bool foundTaxCalculation = false;
|
||||
bool foundAuditTrail = false;
|
||||
bool foundWorkflowCompleted = false;
|
||||
List<string> outputLines = [];
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
|
||||
if (inputSent)
|
||||
{
|
||||
foundStartedRun |= line.Contains("Started run:", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("[Output]", StringComparison.Ordinal))
|
||||
{
|
||||
foundValidateOutput |= line.Contains("ValidateOrder:", StringComparison.Ordinal) && line.Contains("validated", StringComparison.OrdinalIgnoreCase);
|
||||
foundEnrichOutput |= line.Contains("EnrichOrder:", StringComparison.Ordinal) && line.Contains("enriched", StringComparison.OrdinalIgnoreCase);
|
||||
foundPaymentOutput |= line.Contains("ProcessPayment:", StringComparison.Ordinal) && line.Contains("Payment processed", StringComparison.OrdinalIgnoreCase);
|
||||
foundInvoiceOutput |= line.Contains("GenerateInvoice:", StringComparison.Ordinal) && line.Contains("Invoice complete", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Verify shared state: tax rate was read by ProcessPayment
|
||||
foundTaxCalculation |= line.Contains("tax:", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Verify shared state: audit trail was accumulated across executors
|
||||
foundAuditTrail |= line.Contains("Audit trail:", StringComparison.Ordinal)
|
||||
&& line.Contains("ValidateOrder", StringComparison.Ordinal)
|
||||
&& line.Contains("EnrichOrder", StringComparison.Ordinal)
|
||||
&& line.Contains("ProcessPayment", StringComparison.Ordinal);
|
||||
|
||||
outputLines.Add(line);
|
||||
}
|
||||
|
||||
foundWorkflowCompleted |= line.Contains("DurableWorkflowCompletedEvent", StringComparison.Ordinal)
|
||||
|| line.Contains("Completed:", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("Completed:", StringComparison.Ordinal))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the workflow.");
|
||||
Assert.True(foundStartedRun, "Streaming run was not started.");
|
||||
Assert.True(foundValidateOutput, "ValidateOrder output not found in stream.");
|
||||
Assert.True(foundEnrichOutput, "EnrichOrder output not found in stream.");
|
||||
Assert.True(foundPaymentOutput, "ProcessPayment output not found in stream.");
|
||||
Assert.True(foundInvoiceOutput, "GenerateInvoice output not found in stream.");
|
||||
Assert.True(foundTaxCalculation, "Tax calculation (shared state read) not found.");
|
||||
Assert.True(foundAuditTrail, "Audit trail (shared state accumulation) not found.");
|
||||
Assert.True(foundWorkflowCompleted, "Workflow completion not found in stream.");
|
||||
|
||||
// Verify output ordering: ValidateOrder -> EnrichOrder -> ProcessPayment -> GenerateInvoice
|
||||
int validateIndex = outputLines.FindIndex(l => l.Contains("ValidateOrder:", StringComparison.Ordinal) && l.Contains("validated", StringComparison.OrdinalIgnoreCase));
|
||||
int enrichIndex = outputLines.FindIndex(l => l.Contains("EnrichOrder:", StringComparison.Ordinal));
|
||||
int paymentIndex = outputLines.FindIndex(l => l.Contains("ProcessPayment:", StringComparison.Ordinal));
|
||||
int invoiceIndex = outputLines.FindIndex(l => l.Contains("GenerateInvoice:", StringComparison.Ordinal));
|
||||
|
||||
if (validateIndex >= 0 && enrichIndex >= 0)
|
||||
{
|
||||
Assert.True(validateIndex < enrichIndex, "ValidateOrder output should appear before EnrichOrder.");
|
||||
}
|
||||
|
||||
if (enrichIndex >= 0 && paymentIndex >= 0)
|
||||
{
|
||||
Assert.True(enrichIndex < paymentIndex, "EnrichOrder output should appear before ProcessPayment.");
|
||||
}
|
||||
|
||||
if (paymentIndex >= 0 && invoiceIndex >= 0)
|
||||
{
|
||||
Assert.True(paymentIndex < invoiceIndex, "ProcessPayment output should appear before GenerateInvoice.");
|
||||
}
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
||||
public async Task SubWorkflowsSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "07_SubWorkflows");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
|
||||
{
|
||||
bool inputSent = false;
|
||||
bool foundOrderReceived = false;
|
||||
bool foundValidatePayment = false;
|
||||
bool foundAnalyzePatterns = false;
|
||||
bool foundCalculateRiskScore = false;
|
||||
bool foundChargePayment = false;
|
||||
bool foundSelectCarrier = false;
|
||||
bool foundCreateShipment = false;
|
||||
bool foundOrderCompleted = false;
|
||||
bool foundFraudRiskEvent = false;
|
||||
bool workflowCompleted = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
if (!inputSent && line.Contains("Enter an order ID", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await this.WriteInputAsync(process, "ORD-001", testTimeoutCts.Token);
|
||||
inputSent = true;
|
||||
}
|
||||
|
||||
if (inputSent)
|
||||
{
|
||||
// Main workflow executors
|
||||
foundOrderReceived |= line.Contains("[OrderReceived]", StringComparison.Ordinal);
|
||||
foundOrderCompleted |= line.Contains("[OrderCompleted]", StringComparison.Ordinal);
|
||||
|
||||
// Payment sub-workflow executors
|
||||
foundValidatePayment |= line.Contains("[Payment/ValidatePayment]", StringComparison.Ordinal);
|
||||
foundChargePayment |= line.Contains("[Payment/ChargePayment]", StringComparison.Ordinal);
|
||||
|
||||
// FraudCheck sub-sub-workflow executors (nested inside Payment)
|
||||
foundAnalyzePatterns |= line.Contains("[Payment/FraudCheck/AnalyzePatterns]", StringComparison.Ordinal);
|
||||
foundCalculateRiskScore |= line.Contains("[Payment/FraudCheck/CalculateRiskScore]", StringComparison.Ordinal);
|
||||
|
||||
// Shipping sub-workflow executors
|
||||
foundSelectCarrier |= line.Contains("[Shipping/SelectCarrier]", StringComparison.Ordinal);
|
||||
foundCreateShipment |= line.Contains("[Shipping/CreateShipment]", StringComparison.Ordinal);
|
||||
|
||||
// Custom event from nested sub-workflow (streamed to client)
|
||||
foundFraudRiskEvent |= line.Contains("[Event from sub-workflow] FraudRiskAssessedEvent", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("Order completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
workflowCompleted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(inputSent, "Input was not sent to the workflow.");
|
||||
Assert.True(foundOrderReceived, "OrderReceived executor log not found.");
|
||||
Assert.True(foundValidatePayment, "Payment/ValidatePayment executor log not found.");
|
||||
Assert.True(foundAnalyzePatterns, "Payment/FraudCheck/AnalyzePatterns executor log not found.");
|
||||
Assert.True(foundCalculateRiskScore, "Payment/FraudCheck/CalculateRiskScore executor log not found.");
|
||||
Assert.True(foundChargePayment, "Payment/ChargePayment executor log not found.");
|
||||
Assert.True(foundSelectCarrier, "Shipping/SelectCarrier executor log not found.");
|
||||
Assert.True(foundCreateShipment, "Shipping/CreateShipment executor log not found.");
|
||||
Assert.True(foundOrderCompleted, "OrderCompleted executor log not found.");
|
||||
Assert.True(foundFraudRiskEvent, "FraudRiskAssessedEvent from nested sub-workflow not found.");
|
||||
Assert.True(workflowCompleted, "Workflow did not complete successfully.");
|
||||
|
||||
await this.WriteInputAsync(process, "exit", testTimeoutCts.Token);
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "KeyNotFoundException in workflow execution. See https://github.com/microsoft/agent-framework/issues/6404")]
|
||||
public async Task WorkflowHITLSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "08_WorkflowHITL");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, (process, logs) =>
|
||||
{
|
||||
bool foundStarted = false;
|
||||
bool foundManagerApprovalPause = false;
|
||||
bool foundManagerApprovalInput = false;
|
||||
bool foundManagerResponseSent = false;
|
||||
bool foundBudgetApprovalPause = false;
|
||||
bool foundBudgetResponseSent = false;
|
||||
bool foundComplianceApprovalPause = false;
|
||||
bool foundComplianceResponseSent = false;
|
||||
bool foundWorkflowCompleted = false;
|
||||
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
foundStarted |= line.Contains("Starting expense reimbursement workflow", StringComparison.Ordinal);
|
||||
foundManagerApprovalPause |= line.Contains("Workflow paused at RequestPort: ManagerApproval", StringComparison.Ordinal);
|
||||
foundManagerApprovalInput |= line.Contains("Approval for: Jerry", StringComparison.Ordinal);
|
||||
foundManagerResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundManagerApprovalPause && !foundBudgetApprovalPause && !foundComplianceApprovalPause;
|
||||
foundBudgetApprovalPause |= line.Contains("Workflow paused at RequestPort: BudgetApproval", StringComparison.Ordinal);
|
||||
foundBudgetResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundBudgetApprovalPause;
|
||||
foundComplianceApprovalPause |= line.Contains("Workflow paused at RequestPort: ComplianceApproval", StringComparison.Ordinal);
|
||||
foundComplianceResponseSent |= line.Contains("Response sent: Approved=True", StringComparison.Ordinal) && foundComplianceApprovalPause;
|
||||
|
||||
if (line.Contains("Workflow completed: Expense reimbursed at", StringComparison.Ordinal))
|
||||
{
|
||||
foundWorkflowCompleted = true;
|
||||
break;
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
Assert.True(foundStarted, "Workflow start message not found.");
|
||||
Assert.True(foundManagerApprovalPause, "Manager approval pause not found.");
|
||||
Assert.True(foundManagerApprovalInput, "Manager approval input (Jerry) not found.");
|
||||
Assert.True(foundManagerResponseSent, "Manager approval response not sent.");
|
||||
Assert.True(foundBudgetApprovalPause, "Budget approval pause not found.");
|
||||
Assert.True(foundBudgetResponseSent, "Budget approval response not sent.");
|
||||
Assert.True(foundComplianceApprovalPause, "Compliance approval pause not found.");
|
||||
Assert.True(foundComplianceResponseSent, "Compliance approval response not sent.");
|
||||
Assert.True(foundWorkflowCompleted, "Workflow did not complete successfully.");
|
||||
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
}
|
||||
|
||||
[RetryFact(2, 5000, Skip = "Disabled due to persistent CI failures. See #6732.")]
|
||||
public async Task WorkflowAndAgentsSampleValidationAsync()
|
||||
{
|
||||
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
|
||||
string samplePath = Path.Combine(s_samplesPath, "04_WorkflowAndAgents");
|
||||
|
||||
await this.RunSampleTestAsync(samplePath, (process, logs) =>
|
||||
{
|
||||
// Arrange
|
||||
bool foundDemo1 = false;
|
||||
bool foundBiologistResponse = false;
|
||||
bool foundChemistResponse = false;
|
||||
bool foundDemo2 = false;
|
||||
bool foundPhysicsWorkflow = false;
|
||||
bool foundDemo3 = false;
|
||||
bool foundExpertTeamWorkflow = false;
|
||||
bool foundDemo4 = false;
|
||||
bool foundChemistryWorkflow = false;
|
||||
bool allDemosCompleted = false;
|
||||
|
||||
// Act
|
||||
string? line;
|
||||
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
|
||||
{
|
||||
foundDemo1 |= line.Contains("DEMO 1:", StringComparison.Ordinal);
|
||||
foundBiologistResponse |= line.Contains("Biologist:", StringComparison.Ordinal);
|
||||
foundChemistResponse |= line.Contains("Chemist:", StringComparison.Ordinal);
|
||||
foundDemo2 |= line.Contains("DEMO 2:", StringComparison.Ordinal);
|
||||
foundPhysicsWorkflow |= line.Contains("PhysicsExpertReview", StringComparison.Ordinal);
|
||||
foundDemo3 |= line.Contains("DEMO 3:", StringComparison.Ordinal);
|
||||
foundExpertTeamWorkflow |= line.Contains("ExpertTeamReview", StringComparison.Ordinal);
|
||||
foundDemo4 |= line.Contains("DEMO 4:", StringComparison.Ordinal);
|
||||
foundChemistryWorkflow |= line.Contains("ChemistryExpertReview", StringComparison.Ordinal);
|
||||
|
||||
if (line.Contains("All demos completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
allDemosCompleted = true;
|
||||
break;
|
||||
}
|
||||
|
||||
this.AssertNoError(line);
|
||||
}
|
||||
|
||||
// Assert
|
||||
Assert.True(foundDemo1, "DEMO 1 (Direct Agent Conversation) not found.");
|
||||
Assert.True(foundBiologistResponse, "Biologist agent response not found.");
|
||||
Assert.True(foundChemistResponse, "Chemist agent response not found.");
|
||||
Assert.True(foundDemo2, "DEMO 2 (Single-Agent Workflow) not found.");
|
||||
Assert.True(foundPhysicsWorkflow, "PhysicsExpertReview workflow not found.");
|
||||
Assert.True(foundDemo3, "DEMO 3 (Multi-Agent Workflow) not found.");
|
||||
Assert.True(foundExpertTeamWorkflow, "ExpertTeamReview workflow not found.");
|
||||
Assert.True(foundDemo4, "DEMO 4 (Chemistry Workflow) not found.");
|
||||
Assert.True(foundChemistryWorkflow, "ChemistryExpertReview workflow not found.");
|
||||
Assert.True(allDemosCompleted, "Sample did not complete all demos successfully.");
|
||||
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user