// Copyright (c) Microsoft. All rights reserved. using System.Text; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.Onnx; namespace ChatCompletion; /// /// These examples demonstrate the ways different content types are streamed by Onnx GenAI via the chat completion service. /// public class Onnx_ChatCompletionStreaming(ITestOutputHelper output) : BaseTest(output) { /// /// Streaming chat completion streaming using the service directly. /// /// /// Configuration example: /// /// /// ModelId: /// phi-3 /// /// /// ModelPath: /// D:\huggingface\Phi-3-mini-4k-instruct-onnx\cpu_and_mobile\cpu-int4-rtn-block-32 /// /// /// [Fact] public async Task StreamChatAsync() { Assert.NotNull(TestConfiguration.Onnx.ModelId); // dotnet user-secrets set "Onnx:ModelId" "" Assert.NotNull(TestConfiguration.Onnx.ModelPath); // dotnet user-secrets set "Onnx:ModelPath" "" Console.WriteLine("======== Onnx - Chat Completion Streaming ========"); using var chatService = new OnnxRuntimeGenAIChatCompletionService( modelId: TestConfiguration.Onnx.ModelId, modelPath: TestConfiguration.Onnx.ModelPath); Console.WriteLine("Chat content:"); Console.WriteLine("------------------------"); var chatHistory = new ChatHistory("You are a librarian, expert about books"); OutputLastMessage(chatHistory); // First user message chatHistory.AddUserMessage("Hi, I'm looking for book suggestions"); OutputLastMessage(chatHistory); // First assistant message await StreamMessageOutputAsync(chatService, chatHistory, AuthorRole.Assistant); // Second user message chatHistory.AddUserMessage("I love history and philosophy, I'd like to learn something new about Greece, any suggestion?"); OutputLastMessage(chatHistory); // Second assistant message await StreamMessageOutputAsync(chatService, chatHistory, AuthorRole.Assistant); } /// /// Streaming chat completion using the kernel. /// /// /// Configuration example: /// /// /// ModelId: /// phi-3 /// /// /// ModelPath: /// D:\huggingface\Phi-3-mini-4k-instruct-onnx\cpu_and_mobile\cpu-int4-rtn-block-32 /// /// /// [Fact] public async Task StreamChatPromptAsync() { Assert.NotNull(TestConfiguration.Onnx.ModelId); // dotnet user-secrets set "Onnx:ModelId" "" Assert.NotNull(TestConfiguration.Onnx.ModelPath); // dotnet user-secrets set "Onnx:ModelPath" "" StringBuilder chatPrompt = new(""" You are a librarian, expert about books Hi, I'm looking for book suggestions """); Console.WriteLine("======== Onnx - Chat Completion Streaming ========"); var kernel = Kernel.CreateBuilder() .AddOnnxRuntimeGenAIChatCompletion( modelId: TestConfiguration.Onnx.ModelId, modelPath: TestConfiguration.Onnx.ModelPath) .Build(); var reply = await StreamMessageOutputFromKernelAsync(kernel, chatPrompt.ToString()); chatPrompt.AppendLine($""); chatPrompt.AppendLine("I love history and philosophy, I'd like to learn something new about Greece, any suggestion"); reply = await StreamMessageOutputFromKernelAsync(kernel, chatPrompt.ToString()); Console.WriteLine(reply); DisposeServices(kernel); } /// /// This example demonstrates how the chat completion service streams text content. /// It shows how to access the response update via StreamingChatMessageContent.Content property /// and alternatively via the StreamingChatMessageContent.Items property. /// /// /// Configuration example: /// /// /// ModelId: /// phi-3 /// /// /// ModelPath: /// D:\huggingface\Phi-3-mini-4k-instruct-onnx\cpu_and_mobile\cpu-int4-rtn-block-32 /// /// /// [Fact] public async Task StreamTextFromChatAsync() { Assert.NotNull(TestConfiguration.Onnx.ModelId); // dotnet user-secrets set "Onnx:ModelId" "" Assert.NotNull(TestConfiguration.Onnx.ModelPath); // dotnet user-secrets set "Onnx:ModelPath" "" Console.WriteLine("======== Stream Text from Chat Content ========"); // Create chat completion service using var chatService = new OnnxRuntimeGenAIChatCompletionService( modelId: TestConfiguration.Onnx.ModelId, modelPath: TestConfiguration.Onnx.ModelPath); // Create chat history with initial system and user messages ChatHistory chatHistory = new("You are a librarian, an expert on books."); chatHistory.AddUserMessage("Hi, I'm looking for book suggestions."); chatHistory.AddUserMessage("I love history and philosophy. I'd like to learn something new about Greece, any suggestion?"); // Start streaming chat based on the chat history await foreach (StreamingChatMessageContent chatUpdate in chatService.GetStreamingChatMessageContentsAsync(chatHistory)) { // Access the response update via StreamingChatMessageContent.Content property Console.Write(chatUpdate.Content); // Alternatively, the response update can be accessed via the StreamingChatMessageContent.Items property Console.Write(chatUpdate.Items.OfType().FirstOrDefault()); } } private async Task StreamMessageOutputAsync(OnnxRuntimeGenAIChatCompletionService chatCompletionService, ChatHistory chatHistory, AuthorRole authorRole) { bool roleWritten = false; string fullMessage = string.Empty; await foreach (var chatUpdate in chatCompletionService.GetStreamingChatMessageContentsAsync(chatHistory)) { if (!roleWritten && chatUpdate.Role.HasValue) { Console.Write($"{chatUpdate.Role.Value}: {chatUpdate.Content}"); roleWritten = true; } if (chatUpdate.Content is { Length: > 0 }) { fullMessage += chatUpdate.Content; Console.Write(chatUpdate.Content); } } Console.WriteLine("\n------------------------"); chatHistory.AddMessage(authorRole, fullMessage); } private async Task StreamMessageOutputFromKernelAsync(Kernel kernel, string prompt) { bool roleWritten = false; string fullMessage = string.Empty; await foreach (var chatUpdate in kernel.InvokePromptStreamingAsync(prompt)) { if (!roleWritten && chatUpdate.Role.HasValue) { Console.Write($"{chatUpdate.Role.Value}: {chatUpdate.Content}"); roleWritten = true; } if (chatUpdate.Content is { Length: > 0 }) { fullMessage += chatUpdate.Content; Console.Write(chatUpdate.Content); } } Console.WriteLine("\n------------------------"); return fullMessage; } /// /// To avoid any potential memory leak all disposable services created by the kernel are disposed. /// /// Target kernel private static void DisposeServices(Kernel kernel) { foreach (var target in kernel .GetAllServices() .OfType()) { target.Dispose(); } } }