// Copyright (c) Microsoft. All rights reserved. using System.Reflection; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using Microsoft.SemanticKernel.Connectors.OpenAI; using OpenAI.Chat; using Resources; namespace ChatCompletion; /// /// These examples demonstrate how to use audio input and output with OpenAI Chat Completion /// /// /// Currently, audio input and output is only supported with the following models: /// /// gpt-4o-audio-preview /// /// The sample demonstrates: /// /// How to send audio input to the model /// How to receive both text and audio output from the model /// How to save and process the audio response /// /// public class OpenAI_ChatCompletionWithAudio(ITestOutputHelper output) : BaseTest(output) { /// /// This example demonstrates how to use audio input and receive both text and audio output from the model. /// /// /// This sample shows: /// /// Loading audio data from a resource file /// Configuring the chat completion service with audio options /// Enabling both text and audio response modalities /// Extracting and saving the audio response to a file /// Accessing the transcript metadata from the audio response /// /// [Fact] public async Task UsingChatCompletionWithLocalInputAudioAndOutputAudio() { Console.WriteLine($"======== Open AI - {nameof(UsingChatCompletionWithLocalInputAudioAndOutputAudio)} ========\n"); var audioBytes = await EmbeddedResource.ReadAllAsync("test_audio.wav"); var kernel = Kernel.CreateBuilder() .AddOpenAIChatCompletion("gpt-4o-audio-preview", TestConfiguration.OpenAI.ApiKey) .Build(); var chatCompletionService = kernel.GetRequiredService(); var settings = new OpenAIPromptExecutionSettings { Audio = new ChatAudioOptions(ChatOutputAudioVoice.Shimmer, ChatOutputAudioFormat.Mp3), Modalities = ChatResponseModalities.Text | ChatResponseModalities.Audio }; var chatHistory = new ChatHistory("You are a friendly assistant."); chatHistory.AddUserMessage([new AudioContent(audioBytes, "audio/wav")]); var result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings); // Now we need to get the audio content from the result var audioReply = result.Items.First(i => i is AudioContent) as AudioContent; var currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!; var audioFile = Path.Combine(currentDirectory, "audio_output.mp3"); if (File.Exists(audioFile)) { File.Delete(audioFile); } File.WriteAllBytes(audioFile, audioReply!.Data!.Value.ToArray()); Console.WriteLine($"Generated audio: {new Uri(audioFile).AbsoluteUri}"); Console.WriteLine($"Transcript: {audioReply.Metadata!["Transcript"]}"); } /// /// This example demonstrates how to use audio input and receive only text output from the model. /// /// /// This sample shows: /// /// Loading audio data from a resource file /// Configuring the chat completion service with audio options /// Setting response modalities to Text only /// Processing the text response from the model /// /// [Fact] public async Task UsingChatCompletionWithLocalInputAudioAndTextOutput() { Console.WriteLine($"======== Open AI - {nameof(UsingChatCompletionWithLocalInputAudioAndTextOutput)} ========\n"); var audioBytes = await EmbeddedResource.ReadAllAsync("test_audio.wav"); var kernel = Kernel.CreateBuilder() .AddOpenAIChatCompletion("gpt-4o-audio-preview", TestConfiguration.OpenAI.ApiKey) .Build(); var chatCompletionService = kernel.GetRequiredService(); var settings = new OpenAIPromptExecutionSettings { Audio = new ChatAudioOptions(ChatOutputAudioVoice.Shimmer, ChatOutputAudioFormat.Mp3), Modalities = ChatResponseModalities.Text }; var chatHistory = new ChatHistory("You are a friendly assistant."); chatHistory.AddUserMessage([new AudioContent(audioBytes, "audio/wav")]); var result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings); // Now we need to get the audio content from the result Console.WriteLine($"Assistant > {result}"); } }