chore: import upstream snapshot with attribution
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
dotnet-build-and-test / dotnet-test-functions (push) Has been cancelled
dotnet-build-and-test / paths-filter (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Debug, windows-latest, net9.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, ubuntu-latest, net8.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-build (Release, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, ubuntu-latest, net10.0) (push) Has been cancelled
dotnet-build-and-test / dotnet-test (Release, integration, true, windows-latest, net472) (push) Has been cancelled
dotnet-build-and-test / dotnet-foundry-hosted-it (push) Has been cancelled
dotnet-build-and-test / dotnet-build-and-test-check (push) Has been cancelled
dotnet-build-and-test / Integration Test Report (push) Has been cancelled
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
This commit is contained in:
+105
@@ -0,0 +1,105 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
|
||||
using Microsoft.Agents.ObjectModel;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Shared.IntegrationTests;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for workflow tests.
|
||||
/// </summary>
|
||||
public abstract class IntegrationTest : IDisposable
|
||||
{
|
||||
protected IConfigurationRoot Configuration => field ??= InitializeConfig();
|
||||
|
||||
public Uri TestEndpoint { get; }
|
||||
|
||||
public TestOutputAdapter Output { get; }
|
||||
|
||||
protected IntegrationTest(ITestOutputHelper output)
|
||||
{
|
||||
this.Output = new TestOutputAdapter(output);
|
||||
this.TestEndpoint =
|
||||
new Uri(
|
||||
this.Configuration?[TestSettings.AzureAIProjectEndpoint] ??
|
||||
throw new InvalidOperationException($"Undefined configuration setting: {TestSettings.AzureAIProjectEndpoint}"));
|
||||
Console.SetOut(this.Output);
|
||||
SetProduct();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
this.Dispose(isDisposing: true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool isDisposing)
|
||||
{
|
||||
if (isDisposing)
|
||||
{
|
||||
this.Output.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
protected static void SetProduct()
|
||||
{
|
||||
if (!ProductContext.IsLocalScopeSupported())
|
||||
{
|
||||
ProductContext.SetContext(Product.Foundry);
|
||||
}
|
||||
}
|
||||
|
||||
internal static string FormatVariablePath(string variableName, string? scope = null) => $"{scope ?? WorkflowFormulaState.DefaultScopeName}.{variableName}";
|
||||
|
||||
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation = false, params IEnumerable<AIFunction> functionTools)
|
||||
{
|
||||
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, httpRequestHandler: null, functionTools).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IMcpToolHandler? mcpToolProvider, params IEnumerable<AIFunction> functionTools)
|
||||
{
|
||||
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider, httpRequestHandler: null, functionTools).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IHttpRequestHandler? httpRequestHandler, params IEnumerable<AIFunction> functionTools)
|
||||
{
|
||||
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, httpRequestHandler, functionTools).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IMcpToolHandler? mcpToolProvider, IHttpRequestHandler? httpRequestHandler, params IEnumerable<AIFunction> functionTools)
|
||||
{
|
||||
AzureAgentProvider agentProvider =
|
||||
new(this.TestEndpoint, TestAzureCliCredentials.CreateAzureCliCredential())
|
||||
{
|
||||
Functions = functionTools,
|
||||
};
|
||||
|
||||
string? conversationId = null;
|
||||
if (externalConversation)
|
||||
{
|
||||
conversationId = await agentProvider.CreateConversationAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return
|
||||
new DeclarativeWorkflowOptions(agentProvider)
|
||||
{
|
||||
ConversationId = conversationId,
|
||||
LoggerFactory = this.Output,
|
||||
McpToolHandler = mcpToolProvider,
|
||||
HttpRequestHandler = httpRequestHandler,
|
||||
};
|
||||
}
|
||||
|
||||
private static IConfigurationRoot InitializeConfig() =>
|
||||
new ConfigurationBuilder()
|
||||
.AddEnvironmentVariables()
|
||||
.AddUserSecrets(Assembly.GetExecutingAssembly())
|
||||
.Build();
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
public sealed class TestOutputAdapter(ITestOutputHelper output) : TextWriter, ILogger, ILoggerFactory
|
||||
{
|
||||
private readonly Stack<string> _scopes = [];
|
||||
|
||||
public override Encoding Encoding { get; } = Encoding.UTF8;
|
||||
|
||||
public void AddProvider(ILoggerProvider provider) => throw new NotSupportedException();
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => this;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public override void WriteLine(object? value) => this.SafeWrite($"{value}");
|
||||
|
||||
public override void WriteLine(string? format, params object?[] arg) => this.SafeWrite(string.Format(format ?? string.Empty, arg));
|
||||
|
||||
public override void WriteLine(string? value) => this.SafeWrite(value ?? string.Empty);
|
||||
|
||||
public override void Write(object? value) => this.SafeWrite($"{value}");
|
||||
|
||||
public override void Write(char[]? buffer) => this.SafeWrite(new string(buffer));
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state) where TState : notnull
|
||||
{
|
||||
this._scopes.Push($"{state}");
|
||||
return new LoggerScope(() => this._scopes.Pop());
|
||||
}
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
string message = formatter(state, exception);
|
||||
string scope = this._scopes.Count > 0 ? $"[{this._scopes.Peek()}] " : string.Empty;
|
||||
output.WriteLine($"{scope}{message}");
|
||||
}
|
||||
|
||||
private void SafeWrite(string value)
|
||||
{
|
||||
try
|
||||
{
|
||||
output.WriteLine(value ?? string.Empty);
|
||||
}
|
||||
catch (InvalidOperationException exception) when (exception.Message == "There is no currently active test.")
|
||||
{
|
||||
// This exception is thrown when the test output is accessed outside of a test context.
|
||||
// We can ignore it since we are not in a test context.
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class LoggerScope(Action action) : IDisposable
|
||||
{
|
||||
private bool _disposed;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (!this._disposed)
|
||||
{
|
||||
action.Invoke();
|
||||
this._disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
public sealed class Testcase
|
||||
{
|
||||
[JsonConstructor]
|
||||
public Testcase(string description, TestcaseSetup setup, TestcaseValidation validation)
|
||||
{
|
||||
this.Description = description;
|
||||
this.Setup = setup;
|
||||
this.Validation = validation;
|
||||
}
|
||||
|
||||
public string Description { get; }
|
||||
|
||||
public TestcaseSetup Setup { get; }
|
||||
|
||||
public TestcaseValidation Validation { get; }
|
||||
}
|
||||
|
||||
public sealed class TestcaseSetup
|
||||
{
|
||||
[JsonConstructor]
|
||||
public TestcaseSetup(TestcaseInput input)
|
||||
{
|
||||
this.Input = input;
|
||||
}
|
||||
public TestcaseInput Input { get; }
|
||||
public IList<TestcaseInput> Responses { get; init; } = [];
|
||||
}
|
||||
|
||||
public sealed class TestcaseInput
|
||||
{
|
||||
[JsonConstructor]
|
||||
public TestcaseInput(string type, string value)
|
||||
{
|
||||
this.Type = type;
|
||||
this.Value = value;
|
||||
}
|
||||
|
||||
public string Type { get; }
|
||||
public string Value { get; }
|
||||
}
|
||||
|
||||
public sealed class TestcaseValidation
|
||||
{
|
||||
[JsonConstructor]
|
||||
public TestcaseValidation(int conversationCount, int minActionCount, int minResponseCount)
|
||||
{
|
||||
this.ConversationCount = conversationCount;
|
||||
this.MinActionCount = minActionCount;
|
||||
this.MinResponseCount = minResponseCount;
|
||||
}
|
||||
|
||||
public TestcaseValidationActions Actions { get; init; } = TestcaseValidationActions.Empty;
|
||||
public int ConversationCount { get; }
|
||||
public int MinActionCount { get; }
|
||||
// Default expectation is MinActionCount when not defined
|
||||
public int? MaxActionCount { get; init; }
|
||||
// Default expectation is MinResponseCount when not defined
|
||||
public int? MinMessageCount { get; init; }
|
||||
// Default expectation is MaxResponseCount when not defined
|
||||
public int? MaxMessageCount { get; init; }
|
||||
public int MinResponseCount { get; }
|
||||
// Default expectation is MinResponseCount when not defined
|
||||
public int? MaxResponseCount { get; init; }
|
||||
}
|
||||
|
||||
public sealed class TestcaseValidationActions
|
||||
{
|
||||
public static TestcaseValidationActions Empty { get; } = new([]);
|
||||
|
||||
[JsonConstructor]
|
||||
public TestcaseValidationActions(IList<string> start)
|
||||
{
|
||||
this.Start = start;
|
||||
}
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Start { get; }
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Repeat { get; init; } = [];
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
|
||||
public IList<string> Final { get; init; } = [];
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
internal sealed class WorkflowEvents
|
||||
{
|
||||
public WorkflowEvents(IReadOnlyList<WorkflowEvent> workflowEvents)
|
||||
{
|
||||
this.Events = workflowEvents;
|
||||
this.EventCounts = workflowEvents.GroupBy(e => e.GetType()).ToDictionary(e => e.Key, e => e.Count());
|
||||
this.ActionInvokeEvents = workflowEvents.OfType<DeclarativeActionInvokedEvent>().ToList();
|
||||
this.ActionCompleteEvents = workflowEvents.OfType<DeclarativeActionCompletedEvent>().ToList();
|
||||
this.ConversationEvents = workflowEvents.OfType<ConversationUpdateEvent>().ToList();
|
||||
this.ExecutorInvokeEvents = workflowEvents.OfType<ExecutorInvokedEvent>().ToList();
|
||||
this.ExecutorCompleteEvents = workflowEvents.OfType<ExecutorCompletedEvent>().ToList();
|
||||
this.InputEvents = workflowEvents.OfType<RequestInfoEvent>().ToList();
|
||||
this.AgentResponseEvents = workflowEvents.OfType<AgentResponseEvent>().ToList();
|
||||
}
|
||||
|
||||
public IReadOnlyList<WorkflowEvent> Events { get; }
|
||||
public IReadOnlyDictionary<Type, int> EventCounts { get; }
|
||||
public IReadOnlyList<ConversationUpdateEvent> ConversationEvents { get; }
|
||||
public IReadOnlyList<DeclarativeActionInvokedEvent> ActionInvokeEvents { get; }
|
||||
public IReadOnlyList<DeclarativeActionCompletedEvent> ActionCompleteEvents { get; }
|
||||
public IReadOnlyList<ExecutorInvokedEvent> ExecutorInvokeEvents { get; }
|
||||
public IReadOnlyList<ExecutorCompletedEvent> ExecutorCompleteEvents { get; }
|
||||
public IReadOnlyList<RequestInfoEvent> InputEvents { get; }
|
||||
public IReadOnlyList<AgentResponseEvent> AgentResponseEvents { get; }
|
||||
}
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Agents.AI.Workflows.Checkpointing;
|
||||
using Microsoft.Agents.AI.Workflows.Declarative.Events;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Xunit.Sdk;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
internal sealed class WorkflowHarness(Workflow workflow, string runId)
|
||||
{
|
||||
private CheckpointManager? _checkpointManager;
|
||||
private CheckpointInfo? _lastCheckpoint;
|
||||
|
||||
public async Task<WorkflowEvents> RunTestcaseAsync<TInput>(Testcase testcase, TInput input, bool useJson = false) where TInput : notnull
|
||||
{
|
||||
WorkflowEvents workflowEvents = await this.RunWorkflowAsync(input, useJson);
|
||||
int requestCount = workflowEvents.InputEvents.Count;
|
||||
int responseCount = 0;
|
||||
while (requestCount > responseCount)
|
||||
{
|
||||
ExternalRequest request = workflowEvents.InputEvents[workflowEvents.InputEvents.Count - 1].Request;
|
||||
Assert.NotNull(testcase.Setup.Responses);
|
||||
Assert.NotEmpty(testcase.Setup.Responses);
|
||||
string inputText = testcase.Setup.Responses[responseCount].Value;
|
||||
Console.WriteLine($"ID: {request.RequestId}");
|
||||
Console.WriteLine($"INPUT: {inputText}");
|
||||
++responseCount;
|
||||
ExternalResponse response = request.CreateResponse(new ExternalInputResponse(new ChatMessage(ChatRole.User, inputText)));
|
||||
WorkflowEvents runEvents = await this.ResumeAsync(response).ConfigureAwait(false);
|
||||
workflowEvents = new WorkflowEvents([.. workflowEvents.Events, .. runEvents.Events]);
|
||||
requestCount = workflowEvents.InputEvents.Count;
|
||||
}
|
||||
|
||||
return workflowEvents;
|
||||
}
|
||||
|
||||
public async Task<WorkflowEvents> RunWorkflowAsync<TInput>(TInput input, bool useJson = false) where TInput : notnull
|
||||
{
|
||||
Console.WriteLine("RUNNING WORKFLOW...");
|
||||
StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, input, this.GetCheckpointManager(useJson), runId);
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await MonitorAndDisposeWorkflowRunAsync(run).ToArrayAsync();
|
||||
this._lastCheckpoint = workflowEvents.OfType<SuperStepCompletedEvent>().LastOrDefault()?.CompletionInfo?.Checkpoint;
|
||||
return new WorkflowEvents(workflowEvents);
|
||||
}
|
||||
|
||||
public async Task<WorkflowEvents> ResumeAsync(ExternalResponse response)
|
||||
{
|
||||
Console.WriteLine("\nRESUMING WORKFLOW...");
|
||||
Assert.NotNull(this._lastCheckpoint);
|
||||
StreamingRun run = await InProcessExecution.ResumeStreamingAsync(workflow, this._lastCheckpoint, this.GetCheckpointManager());
|
||||
IReadOnlyList<WorkflowEvent> workflowEvents = await MonitorAndDisposeWorkflowRunAsync(run, response).ToArrayAsync();
|
||||
this._lastCheckpoint = workflowEvents.OfType<SuperStepCompletedEvent>().LastOrDefault()?.CompletionInfo?.Checkpoint;
|
||||
return new WorkflowEvents(workflowEvents);
|
||||
}
|
||||
|
||||
private CheckpointManager GetCheckpointManager(bool useJson = false)
|
||||
{
|
||||
if (useJson && this._checkpointManager is null)
|
||||
{
|
||||
DirectoryInfo checkpointFolder = Directory.CreateDirectory(Path.Combine(".", $"chk-{DateTime.Now:yyMMdd-hhmmss-ff}"));
|
||||
this._checkpointManager = CheckpointManager.CreateJson(new FileSystemJsonCheckpointStore(checkpointFolder));
|
||||
}
|
||||
else
|
||||
{
|
||||
this._checkpointManager ??= CheckpointManager.CreateInMemory();
|
||||
}
|
||||
|
||||
return this._checkpointManager;
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<WorkflowEvent> MonitorAndDisposeWorkflowRunAsync(StreamingRun run, ExternalResponse? response = null)
|
||||
{
|
||||
await using IAsyncDisposable disposeRun = run;
|
||||
|
||||
if (response is not null)
|
||||
{
|
||||
await run.SendResponseAsync(response).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
bool exitLoop = false;
|
||||
bool hasRequest = false;
|
||||
|
||||
await foreach (WorkflowEvent workflowEvent in run.WatchStreamAsync().ConfigureAwait(false))
|
||||
{
|
||||
switch (workflowEvent)
|
||||
{
|
||||
case SuperStepCompletedEvent:
|
||||
if (hasRequest)
|
||||
{
|
||||
exitLoop = true;
|
||||
}
|
||||
break;
|
||||
case RequestInfoEvent requestInfo:
|
||||
Console.WriteLine($"REQUEST #{requestInfo.Request.RequestId}");
|
||||
// Only count as a new request if it's not the one we're responding to
|
||||
if (response is null || requestInfo.Request.RequestId != response.RequestId)
|
||||
{
|
||||
hasRequest = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// This is a republished event for the request we're already responding to
|
||||
// (emitted by RepublishUnservicedRequestsAsync during checkpoint resume).
|
||||
// Skip yielding it so downstream code doesn't treat it as a new pending request.
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
|
||||
case ConversationUpdateEvent conversationEvent:
|
||||
Console.WriteLine($"CONVERSATION: {conversationEvent.ConversationId}");
|
||||
break;
|
||||
|
||||
case ExecutorFailedEvent failureEvent:
|
||||
Console.WriteLine($"Executor failed [{failureEvent.ExecutorId}]: {failureEvent.Data?.Message ?? "Unknown"}");
|
||||
break;
|
||||
|
||||
case WorkflowErrorEvent errorEvent:
|
||||
throw errorEvent.Data as Exception ?? new XunitException("Unexpected failure...");
|
||||
|
||||
case ExecutorInvokedEvent executorInvokeEvent:
|
||||
Console.WriteLine($"EXEC: {executorInvokeEvent.ExecutorId}");
|
||||
break;
|
||||
|
||||
case DeclarativeActionInvokedEvent actionInvokeEvent:
|
||||
Console.WriteLine($"ACTION: {actionInvokeEvent.ActionId} [{actionInvokeEvent.ActionType}]");
|
||||
break;
|
||||
|
||||
case AgentResponseEvent responseEvent:
|
||||
if (!string.IsNullOrEmpty(responseEvent.Response.Text))
|
||||
{
|
||||
Console.WriteLine($"AGENT: {responseEvent.Response.AgentId}: {responseEvent.Response.Text}");
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (FunctionCallContent toolCall in responseEvent.Response.Messages.SelectMany(m => m.Contents.OfType<FunctionCallContent>()))
|
||||
{
|
||||
Console.WriteLine($"TOOL: {toolCall.Name} [{responseEvent.Response.AgentId}]");
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
yield return workflowEvent;
|
||||
|
||||
if (exitLoop)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("SUSPENDING WORKFLOW...\n");
|
||||
}
|
||||
}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Xunit.Sdk;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for workflow tests.
|
||||
/// </summary>
|
||||
public abstract class WorkflowTest(ITestOutputHelper output) : IntegrationTest(output)
|
||||
{
|
||||
protected abstract Task RunAndVerifyAsync<TInput>(
|
||||
Testcase testcase,
|
||||
string workflowPath,
|
||||
DeclarativeWorkflowOptions workflowOptions,
|
||||
TInput input,
|
||||
bool useJsonCheckpoint) where TInput : notnull;
|
||||
|
||||
protected Task RunWorkflowAsync(
|
||||
string workflowPath,
|
||||
string testcaseFileName,
|
||||
bool externalConversation = false,
|
||||
bool useJsonCheckpoint = false)
|
||||
{
|
||||
this.Output.WriteLine($"WORKFLOW: {workflowPath}");
|
||||
this.Output.WriteLine($"TESTCASE: {testcaseFileName}");
|
||||
|
||||
Testcase testcase = ReadTestcase(testcaseFileName);
|
||||
|
||||
this.Output.WriteLine($" {testcase.Description}");
|
||||
|
||||
return
|
||||
testcase.Setup.Input.Type switch
|
||||
{
|
||||
nameof(ChatMessage) => TestWorkflowAsync<ChatMessage>(),
|
||||
nameof(String) => TestWorkflowAsync<string>(),
|
||||
_ => throw new NotSupportedException($"Input type '{testcase.Setup.Input.Type}' is not supported."),
|
||||
};
|
||||
|
||||
async Task TestWorkflowAsync<TInput>() where TInput : notnull
|
||||
{
|
||||
this.Output.WriteLine($"INPUT: {testcase.Setup.Input.Value}");
|
||||
|
||||
DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(externalConversation).ConfigureAwait(false);
|
||||
|
||||
TInput input = (TInput)GetInput<TInput>(testcase);
|
||||
|
||||
await this.RunAndVerifyAsync(testcase, workflowPath, workflowOptions, input, useJsonCheckpoint);
|
||||
}
|
||||
}
|
||||
|
||||
protected static string? GetConversationId(string? conversationId, IReadOnlyList<ConversationUpdateEvent> conversationEvents)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(conversationId))
|
||||
{
|
||||
return conversationId;
|
||||
}
|
||||
|
||||
if (conversationEvents.Count > 0)
|
||||
{
|
||||
return conversationEvents.SingleOrDefault(conversationEvent => conversationEvent.IsWorkflow)?.ConversationId;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
protected static Testcase ReadTestcase(string testcaseFileName)
|
||||
{
|
||||
string testcaseJson = File.ReadAllText(Path.Combine("Testcases", testcaseFileName));
|
||||
Testcase? testcase = JsonSerializer.Deserialize<Testcase>(testcaseJson, s_jsonSerializerOptions);
|
||||
Assert.NotNull(testcase);
|
||||
return testcase;
|
||||
}
|
||||
|
||||
private static object GetInput<TInput>(Testcase testcase) where TInput : notnull =>
|
||||
testcase.Setup.Input.Type switch
|
||||
{
|
||||
nameof(ChatMessage) => new ChatMessage(ChatRole.User, testcase.Setup.Input.Value),
|
||||
nameof(String) => testcase.Setup.Input.Value,
|
||||
_ => throw new NotSupportedException($"Input type '{testcase.Setup.Input.Type}' is not supported."),
|
||||
};
|
||||
|
||||
internal static string GetRepoFolder()
|
||||
{
|
||||
DirectoryInfo? current = new(Directory.GetCurrentDirectory());
|
||||
|
||||
while (current is not null)
|
||||
{
|
||||
if (Directory.Exists(Path.Combine(current.FullName, "declarative-agents", "workflow-samples")))
|
||||
{
|
||||
return current.FullName;
|
||||
}
|
||||
|
||||
current = current.Parent;
|
||||
}
|
||||
|
||||
throw new XunitException("Unable to locate repository root folder.");
|
||||
}
|
||||
|
||||
protected static class AssertWorkflow
|
||||
{
|
||||
public static void Conversation(IReadOnlyList<ConversationUpdateEvent> conversationEvents, Testcase testcase)
|
||||
{
|
||||
Assert.Equal(testcase.Validation.ConversationCount, conversationEvents.Count);
|
||||
}
|
||||
|
||||
// "isCompletion" adjusts validation logic to account for when condition completion is not experienced due to goto. Remove this test logic once addressed.
|
||||
public static void EventCounts(int actualCount, Testcase testcase, bool isCompletion = false)
|
||||
{
|
||||
Assert.True(actualCount + (isCompletion ? 1 : 0) >= testcase.Validation.MinActionCount, $"Event count less than expected: {testcase.Validation.MinActionCount} (Actual: {actualCount}).");
|
||||
if (testcase.Validation.MaxActionCount != -1)
|
||||
{
|
||||
int maxExpectedCount = testcase.Validation.MaxActionCount ?? testcase.Validation.MinActionCount;
|
||||
Assert.True(actualCount <= maxExpectedCount, $"Event count greater than expected: {maxExpectedCount} (Actual: {actualCount}).");
|
||||
}
|
||||
}
|
||||
|
||||
public static void Responses(IReadOnlyList<AgentResponseEvent> responseEvents, Testcase testcase)
|
||||
{
|
||||
Assert.True(responseEvents.Count >= testcase.Validation.MinResponseCount, $"Response count less than expected: {testcase.Validation.MinResponseCount} (Actual: {responseEvents.Count})");
|
||||
if (testcase.Validation.MaxResponseCount != -1)
|
||||
{
|
||||
int maxExpectedCount = testcase.Validation.MaxResponseCount ?? testcase.Validation.MinResponseCount;
|
||||
Assert.True(responseEvents.Count <= maxExpectedCount, $"Response count greater than expected: {maxExpectedCount} (Actual: {responseEvents.Count}).");
|
||||
}
|
||||
}
|
||||
|
||||
public static async ValueTask MessagesAsync(string? conversationId, Testcase testcase, ResponseAgentProvider agentProvider)
|
||||
{
|
||||
int minExpectedCount = testcase.Validation.MinMessageCount ?? testcase.Validation.MinResponseCount;
|
||||
int maxExpectedCount = testcase.Validation.MaxMessageCount ?? testcase.Validation.MaxResponseCount ?? minExpectedCount;
|
||||
int messageCount = 0;
|
||||
if (!string.IsNullOrEmpty(conversationId))
|
||||
{
|
||||
messageCount = await agentProvider.GetMessagesAsync(conversationId).CountAsync();
|
||||
}
|
||||
|
||||
++minExpectedCount;
|
||||
Assert.True(messageCount >= minExpectedCount, $"Workflow message count less than expected: {minExpectedCount} (Actual: {messageCount}).");
|
||||
if (maxExpectedCount != -1)
|
||||
{
|
||||
++maxExpectedCount;
|
||||
Assert.True(messageCount <= maxExpectedCount, $"Workflow message count greater than expected: {maxExpectedCount} (Actual: {messageCount}).");
|
||||
}
|
||||
}
|
||||
|
||||
internal static void EventSequence(IEnumerable<string> sourceIds, Testcase testcase)
|
||||
{
|
||||
string lastId = string.Empty;
|
||||
Queue<string> startIds = [];
|
||||
Queue<string> repeatIds = [];
|
||||
bool validateStart = false;
|
||||
bool validateRepeat = false;
|
||||
foreach (string sourceId in sourceIds)
|
||||
{
|
||||
if (!validateStart && testcase.Validation.Actions.Start.Count > 0)
|
||||
{
|
||||
if (testcase.Validation.Actions.Start.Count > 0 &&
|
||||
startIds.Count == 0 &&
|
||||
sourceId.Equals(testcase.Validation.Actions.Start[0], StringComparison.Ordinal))
|
||||
{
|
||||
// Initialize start sequence
|
||||
startIds = new(testcase.Validation.Actions.Start);
|
||||
}
|
||||
|
||||
// Verify start sequence
|
||||
if (startIds.Count > 0)
|
||||
{
|
||||
Assert.Equal(startIds.Dequeue(), sourceId);
|
||||
validateStart = startIds.Count == 0;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (testcase.Validation.Actions.Repeat.Count > 0 &&
|
||||
repeatIds.Count == 0 &&
|
||||
sourceId.Equals(testcase.Validation.Actions.Repeat[0], StringComparison.Ordinal))
|
||||
{
|
||||
// Initialize repeat sequence
|
||||
repeatIds = new(testcase.Validation.Actions.Repeat);
|
||||
}
|
||||
// Verify repeat sequence
|
||||
if (repeatIds.Count > 0)
|
||||
{
|
||||
Assert.Equal(repeatIds.Dequeue(), sourceId);
|
||||
validateRepeat = true;
|
||||
}
|
||||
}
|
||||
lastId = sourceId;
|
||||
}
|
||||
|
||||
Assert.Equal(testcase.Validation.Actions.Start.Count > 0, validateStart);
|
||||
Assert.Equal(testcase.Validation.Actions.Repeat.Count > 0, validateRepeat);
|
||||
|
||||
Assert.NotEmpty(lastId);
|
||||
HashSet<string> finalIds = [.. testcase.Validation.Actions.Final];
|
||||
Assert.Contains(lastId, finalIds);
|
||||
}
|
||||
}
|
||||
|
||||
protected static readonly JsonSerializerOptions s_jsonSerializerOptions = new()
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
WriteIndented = true,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user