chore: import upstream snapshot with attribution

dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs

@@ -0,0 +1,170 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Net;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using UglyToad.PdfPig;
+using UglyToad.PdfPig.Content;
+using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter;
+
+namespace VectorStoreRAG;
+
+/// <summary>
+/// Class that loads text from a PDF file into a vector store.
+/// </summary>
+/// <typeparam name="TKey">The type of the data model key.</typeparam>
+/// <param name="uniqueKeyGenerator">A function to generate unique keys with.</param>
+/// <param name="vectorStoreRecordCollection">The collection to load the data into.</param>
+/// <param name="chatCompletionService">The chat completion service to use for generating text from images.</param>
+internal sealed class DataLoader<TKey>(
+    UniqueKeyGenerator<TKey> uniqueKeyGenerator,
+    VectorStoreCollection<TKey, TextSnippet<TKey>> vectorStoreRecordCollection,
+    IChatCompletionService chatCompletionService) : IDataLoader where TKey : notnull
+{
+    /// <inheritdoc/>
+    public async Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken)
+    {
+        // Create the collection if it doesn't exist.
+        await vectorStoreRecordCollection.EnsureCollectionExistsAsync(cancellationToken).ConfigureAwait(false);
+
+        // Load the text and images from the PDF file and split them into batches.
+        var sections = LoadTextAndImages(pdfPath, cancellationToken);
+        var batches = sections.Chunk(batchSize);
+
+        // Process each batch of content items.
+        foreach (var batch in batches)
+        {
+            // Convert any images to text.
+            var textContentTasks = batch.Select(async content =>
+            {
+                if (content.Text != null)
+                {
+                    return content;
+                }
+
+                var textFromImage = await ConvertImageToTextWithRetryAsync(
+                    chatCompletionService,
+                    content.Image!.Value,
+                    cancellationToken).ConfigureAwait(false);
+                return new RawContent { Text = textFromImage, PageNumber = content.PageNumber };
+            });
+            var textContent = await Task.WhenAll(textContentTasks).ConfigureAwait(false);
+
+            // Map each paragraph to a TextSnippet.
+            var records = textContent.Select(content => new TextSnippet<TKey>
+            {
+                Key = uniqueKeyGenerator.GenerateKey(),
+                // The vector store will automatically generate the embedding for this text.
+                // See the TextEmbedding field on the TextSnippet class.
+                Text = content.Text,
+                ReferenceDescription = $"{new FileInfo(pdfPath).Name}#page={content.PageNumber}",
+                ReferenceLink = $"{new Uri(new FileInfo(pdfPath).FullName).AbsoluteUri}#page={content.PageNumber}",
+            });
+
+            // Upsert the records into the vector store.
+            await vectorStoreRecordCollection.UpsertAsync(records, cancellationToken: cancellationToken).ConfigureAwait(false);
+
+            await Task.Delay(betweenBatchDelayInMs, cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    /// <summary>
+    /// Read the text and images from each page in the provided PDF file.
+    /// </summary>
+    /// <param name="pdfPath">The pdf file to read the text and images from.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>The text and images from the pdf file, plus the page number that each is on.</returns>
+    private static IEnumerable<RawContent> LoadTextAndImages(string pdfPath, CancellationToken cancellationToken)
+    {
+        using (PdfDocument document = PdfDocument.Open(pdfPath))
+        {
+            foreach (Page page in document.GetPages())
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    break;
+                }
+
+                foreach (var image in page.GetImages())
+                {
+                    if (image.TryGetPng(out var png))
+                    {
+                        yield return new RawContent { Image = png, PageNumber = page.Number };
+                    }
+                    else
+                    {
+                        Console.WriteLine($"Unsupported image format on page {page.Number}");
+                    }
+                }
+
+                var blocks = DefaultPageSegmenter.Instance.GetBlocks(page.GetWords());
+                foreach (var block in blocks)
+                {
+                    if (cancellationToken.IsCancellationRequested)
+                    {
+                        break;
+                    }
+
+                    yield return new RawContent { Text = block.Text, PageNumber = page.Number };
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Add a simple retry mechanism to image to text.
+    /// </summary>
+    /// <param name="chatCompletionService">The chat completion service to use for generating text from images.</param>
+    /// <param name="imageBytes">The image to generate the text for.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>The generated text.</returns>
+    private static async Task<string> ConvertImageToTextWithRetryAsync(
+        IChatCompletionService chatCompletionService,
+        ReadOnlyMemory<byte> imageBytes,
+        CancellationToken cancellationToken)
+    {
+        var tries = 0;
+
+        while (true)
+        {
+            try
+            {
+                var chatHistory = new ChatHistory();
+                chatHistory.AddUserMessage([
+                    new TextContent("What’s in this image?"),
+                    new ImageContent(imageBytes, "image/png"),
+                ]);
+                var result = await chatCompletionService.GetChatMessageContentsAsync(chatHistory, cancellationToken: cancellationToken).ConfigureAwait(false);
+                return string.Join("\n", result.Select(x => x.Content));
+            }
+            catch (HttpOperationException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests)
+            {
+                tries++;
+
+                if (tries < 3)
+                {
+                    Console.WriteLine($"Failed to generate text from image. Error: {ex}");
+                    Console.WriteLine("Retrying text to image conversion...");
+                    await Task.Delay(10_000, cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Private model for returning the content items from a PDF file.
+    /// </summary>
+    private sealed class RawContent
+    {
+        public string? Text { get; init; }
+
+        public ReadOnlyMemory<byte>? Image { get; init; }
+
+        public int PageNumber { get; init; }
+    }
+}

dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VectorStoreRAG;
+
+/// <summary>
+/// Interface for loading data into a data store.
+/// </summary>
+internal interface IDataLoader
+{
+    /// <summary>
+    /// Load the text from a PDF file into the data store.
+    /// </summary>
+    /// <param name="pdfPath">The pdf file to load.</param>
+    /// <param name="batchSize">Maximum number of parallel threads to generate embeddings and upload records.</param>
+    /// <param name="betweenBatchDelayInMs">The number of milliseconds to delay between batches to avoid throttling.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>An async task that completes when the loading is complete.</returns>
+    Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken);
+}

dotnet/samples/Demos/VectorStoreRAG/Options/ApplicationConfig.cs

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.Configuration;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Helper class to load all configuration settings for the VectorStoreRAG project.
+/// </summary>
+internal sealed class ApplicationConfig
+{
+    private readonly AzureOpenAIConfig _azureOpenAIConfig;
+    private readonly AzureOpenAIEmbeddingsConfig _azureOpenAIEmbeddingsConfig = new();
+    private readonly OpenAIConfig _openAIConfig = new();
+    private readonly OpenAIEmbeddingsConfig _openAIEmbeddingsConfig = new();
+    private readonly RagConfig _ragConfig = new();
+    private readonly AzureAISearchConfig _azureAISearchConfig = new();
+    private readonly CosmosConfig _cosmosMongoConfig = new();
+    private readonly CosmosConfig _cosmosNoSqlConfig = new();
+    private readonly QdrantConfig _qdrantConfig = new();
+    private readonly RedisConfig _redisConfig = new();
+    private readonly WeaviateConfig _weaviateConfig = new();
+
+    public ApplicationConfig(ConfigurationManager configurationManager)
+    {
+        this._azureOpenAIConfig = new();
+        configurationManager
+            .GetRequiredSection($"AIServices:{AzureOpenAIConfig.ConfigSectionName}")
+            .Bind(this._azureOpenAIConfig);
+        configurationManager
+            .GetRequiredSection($"AIServices:{AzureOpenAIEmbeddingsConfig.ConfigSectionName}")
+            .Bind(this._azureOpenAIEmbeddingsConfig);
+        configurationManager
+            .GetRequiredSection($"AIServices:{OpenAIConfig.ConfigSectionName}")
+            .Bind(this._openAIConfig);
+        configurationManager
+            .GetRequiredSection($"AIServices:{OpenAIEmbeddingsConfig.ConfigSectionName}")
+            .Bind(this._openAIEmbeddingsConfig);
+        configurationManager
+            .GetRequiredSection(RagConfig.ConfigSectionName)
+            .Bind(this._ragConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{AzureAISearchConfig.ConfigSectionName}")
+            .Bind(this._azureAISearchConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{CosmosConfig.MongoConfigSectionName}")
+            .Bind(this._cosmosMongoConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{CosmosConfig.NoSqlConfigSectionName}")
+            .Bind(this._cosmosNoSqlConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{QdrantConfig.ConfigSectionName}")
+            .Bind(this._qdrantConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{RedisConfig.ConfigSectionName}")
+            .Bind(this._redisConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{WeaviateConfig.ConfigSectionName}")
+            .Bind(this._weaviateConfig);
+    }
+
+    public AzureOpenAIConfig AzureOpenAIConfig => this._azureOpenAIConfig;
+
+    public AzureOpenAIEmbeddingsConfig AzureOpenAIEmbeddingsConfig => this._azureOpenAIEmbeddingsConfig;
+
+    public OpenAIConfig OpenAIConfig => this._openAIConfig;
+
+    public OpenAIEmbeddingsConfig OpenAIEmbeddingsConfig => this._openAIEmbeddingsConfig;
+
+    public RagConfig RagConfig => this._ragConfig;
+
+    public AzureAISearchConfig AzureAISearchConfig => this._azureAISearchConfig;
+
+    public CosmosConfig CosmosMongoConfig => this._cosmosMongoConfig;
+
+    public CosmosConfig CosmosNoSqlConfig => this._cosmosNoSqlConfig;
+
+    public QdrantConfig QdrantConfig => this._qdrantConfig;
+
+    public RedisConfig RedisConfig => this._redisConfig;
+
+    public WeaviateConfig WeaviateConfig => this._weaviateConfig;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/AzureAISearchConfig.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Azure AI Search service settings.
+/// </summary>
+internal sealed class AzureAISearchConfig
+{
+    public const string ConfigSectionName = "AzureAISearch";
+
+    [Required]
+    public string Endpoint { get; set; } = string.Empty;
+
+    [Required]
+    public string ApiKey { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/AzureOpenAIConfig.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Azure OpenAI service settings.
+/// </summary>
+internal sealed class AzureOpenAIConfig
+{
+    public const string ConfigSectionName = "AzureOpenAI";
+
+    [Required]
+    public string ChatDeploymentName { get; set; } = string.Empty;
+
+    [Required]
+    public string Endpoint { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/AzureOpenAIEmbeddingsConfig.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Azure OpenAI Embeddings service settings.
+/// </summary>
+internal sealed class AzureOpenAIEmbeddingsConfig
+{
+    public const string ConfigSectionName = "AzureOpenAIEmbeddings";
+
+    [Required]
+    public string DeploymentName { get; set; } = string.Empty;
+
+    [Required]
+    public string Endpoint { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/CosmosConfig.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Azure CosmosDB service settings for use with CosmosMongo and CosmosNoSql.
+/// </summary>
+internal sealed class CosmosConfig
+{
+    public const string MongoConfigSectionName = "CosmosMongoDB";
+    public const string NoSqlConfigSectionName = "CosmosNoSql";
+
+    [Required]
+    public string ConnectionString { get; set; } = string.Empty;
+
+    [Required]
+    public string DatabaseName { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/OpenAIConfig.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// OpenAI service settings.
+/// </summary>
+internal sealed class OpenAIConfig
+{
+    public const string ConfigSectionName = "OpenAI";
+
+    [Required]
+    public string ModelId { get; set; } = string.Empty;
+
+    [Required]
+    public string ApiKey { get; set; } = string.Empty;
+
+    [Required]
+    public string? OrgId { get; set; } = null;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/OpenAIEmbeddingsConfig.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// OpenAI Embeddings service settings.
+/// </summary>
+internal sealed class OpenAIEmbeddingsConfig
+{
+    public const string ConfigSectionName = "OpenAIEmbeddings";
+
+    [Required]
+    public string ModelId { get; set; } = string.Empty;
+
+    [Required]
+    public string ApiKey { get; set; } = string.Empty;
+
+    [Required]
+    public string? OrgId { get; set; } = null;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/QdrantConfig.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Qdrant service settings.
+/// </summary>
+internal sealed class QdrantConfig
+{
+    public const string ConfigSectionName = "Qdrant";
+
+    [Required]
+    public string Host { get; set; } = string.Empty;
+
+    public int Port { get; set; } = 6334;
+
+    public bool Https { get; set; } = false;
+
+    public string ApiKey { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Contains settings to control the RAG experience.
+/// </summary>
+internal sealed class RagConfig
+{
+    public const string ConfigSectionName = "Rag";
+
+    [Required]
+    public string AIChatService { get; set; } = string.Empty;
+
+    [Required]
+    public string AIEmbeddingService { get; set; } = string.Empty;
+
+    [Required]
+    public bool BuildCollection { get; set; } = true;
+
+    [Required]
+    public string CollectionName { get; set; } = string.Empty;
+
+    [Required]
+    public int DataLoadingBatchSize { get; set; } = 2;
+
+    [Required]
+    public int DataLoadingBetweenBatchDelayInMilliseconds { get; set; } = 0;
+
+    [Required]
+    public string[]? PdfFilePaths { get; set; }
+
+    [Required]
+    public string VectorStoreType { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/RedisConfig.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Redis service settings.
+/// </summary>
+internal sealed class RedisConfig
+{
+    public const string ConfigSectionName = "Redis";
+
+    [Required]
+    public string ConnectionConfiguration { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Options/WeaviateConfig.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Weaviate service settings.
+/// </summary>
+internal sealed class WeaviateConfig
+{
+    public const string ConfigSectionName = "Weaviate";
+
+    [Required]
+    public string Endpoint { get; set; } = string.Empty;
+}

dotnet/samples/Demos/VectorStoreRAG/Program.cs

@@ -0,0 +1,154 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Globalization;
+using Azure;
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.SemanticKernel;
+using OpenAI;
+using VectorStoreRAG;
+using VectorStoreRAG.Options;
+
+HostApplicationBuilder builder = Host.CreateApplicationBuilder(args);
+
+// Configure configuration and load the application configuration.
+builder.Configuration.AddUserSecrets<Program>();
+builder.Services.Configure<RagConfig>(builder.Configuration.GetSection(RagConfig.ConfigSectionName));
+var appConfig = new ApplicationConfig(builder.Configuration);
+
+// Create a cancellation token and source to pass to the application service to allow them
+// to request a graceful application shutdown.
+CancellationTokenSource appShutdownCancellationTokenSource = new();
+CancellationToken appShutdownCancellationToken = appShutdownCancellationTokenSource.Token;
+builder.Services.AddKeyedSingleton("AppShutdown", appShutdownCancellationTokenSource);
+
+// Register the kernel with the dependency injection container
+// and add Chat Completion and Text Embedding Generation services.
+var kernelBuilder = builder.Services.AddKernel();
+
+switch (appConfig.RagConfig.AIChatService)
+{
+    case "AzureOpenAI":
+        kernelBuilder.AddAzureOpenAIChatCompletion(
+            appConfig.AzureOpenAIConfig.ChatDeploymentName,
+            appConfig.AzureOpenAIConfig.Endpoint,
+            new AzureCliCredential());
+        break;
+    case "OpenAI":
+        kernelBuilder.AddOpenAIChatCompletion(
+            appConfig.OpenAIConfig.ModelId,
+            appConfig.OpenAIConfig.ApiKey,
+            appConfig.OpenAIConfig.OrgId);
+        break;
+    default:
+        throw new NotSupportedException($"AI Chat Service type '{appConfig.RagConfig.AIChatService}' is not supported.");
+}
+
+switch (appConfig.RagConfig.AIEmbeddingService)
+{
+    case "AzureOpenAIEmbeddings":
+        builder.Services.AddSingleton<IEmbeddingGenerator>(
+            sp => new AzureOpenAIClient(new Uri(appConfig.AzureOpenAIEmbeddingsConfig.Endpoint), new AzureCliCredential())
+                .GetEmbeddingClient(appConfig.AzureOpenAIEmbeddingsConfig.DeploymentName)
+                .AsIEmbeddingGenerator());
+        break;
+    case "OpenAIEmbeddings":
+        builder.Services.AddSingleton<IEmbeddingGenerator>(
+            sp => new OpenAIClient(appConfig.OpenAIEmbeddingsConfig.ApiKey)
+                .GetEmbeddingClient(appConfig.OpenAIEmbeddingsConfig.ModelId)
+                .AsIEmbeddingGenerator());
+        break;
+    default:
+        throw new NotSupportedException($"AI Embedding Service type '{appConfig.RagConfig.AIEmbeddingService}' is not supported.");
+}
+
+// Add the configured vector store record collection type to the
+// dependency injection container.
+switch (appConfig.RagConfig.VectorStoreType)
+{
+    case "AzureAISearch":
+        kernelBuilder.Services.AddAzureAISearchCollection<TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName,
+            new Uri(appConfig.AzureAISearchConfig.Endpoint),
+            new AzureKeyCredential(appConfig.AzureAISearchConfig.ApiKey));
+        break;
+    case "CosmosMongoDB":
+        kernelBuilder.Services.AddCosmosMongoCollection<TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName,
+            appConfig.CosmosMongoConfig.ConnectionString,
+            appConfig.CosmosMongoConfig.DatabaseName);
+        break;
+    case "CosmosNoSql":
+        kernelBuilder.Services.AddCosmosNoSqlCollection<string, TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName,
+            appConfig.CosmosNoSqlConfig.ConnectionString,
+            appConfig.CosmosNoSqlConfig.DatabaseName);
+        break;
+    case "InMemory":
+        kernelBuilder.Services.AddInMemoryVectorStoreRecordCollection<string, TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName);
+        break;
+    case "Qdrant":
+        kernelBuilder.Services.AddQdrantCollection<Guid, TextSnippet<Guid>>(
+            appConfig.RagConfig.CollectionName,
+            appConfig.QdrantConfig.Host,
+            appConfig.QdrantConfig.Port,
+            appConfig.QdrantConfig.Https,
+            appConfig.QdrantConfig.ApiKey);
+        break;
+    case "Redis":
+        kernelBuilder.Services.AddRedisJsonCollection<TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName,
+            appConfig.RedisConfig.ConnectionConfiguration);
+        break;
+    case "Weaviate":
+        kernelBuilder.Services.AddWeaviateCollection<TextSnippet<Guid>>(
+            // Weaviate collection names must start with an upper case letter.
+            char.ToUpper(appConfig.RagConfig.CollectionName[0], CultureInfo.InvariantCulture) + appConfig.RagConfig.CollectionName.Substring(1),
+            endpoint: new Uri(appConfig.WeaviateConfig.Endpoint),
+            apiKey: null);
+        break;
+    default:
+        throw new NotSupportedException($"Vector store type '{appConfig.RagConfig.VectorStoreType}' is not supported.");
+}
+
+// Register all the other required services.
+switch (appConfig.RagConfig.VectorStoreType)
+{
+    case "AzureAISearch":
+    case "CosmosMongoDB":
+    case "CosmosNoSql":
+    case "InMemory":
+    case "Redis":
+        RegisterServices<string>(builder, kernelBuilder, appConfig);
+        break;
+    case "Qdrant":
+    case "Weaviate":
+        RegisterServices<Guid>(builder, kernelBuilder, appConfig);
+        break;
+    default:
+        throw new NotSupportedException($"Vector store type '{appConfig.RagConfig.VectorStoreType}' is not supported.");
+}
+
+// Build and run the host.
+using IHost host = builder.Build();
+await host.RunAsync(appShutdownCancellationToken).ConfigureAwait(false);
+
+static void RegisterServices<TKey>(HostApplicationBuilder builder, IKernelBuilder kernelBuilder, ApplicationConfig vectorStoreRagConfig)
+    where TKey : notnull
+{
+    // Add a text search implementation that uses the registered vector store record collection for search.
+    kernelBuilder.AddVectorStoreTextSearch<TextSnippet<TKey>>();
+
+    // Add the key generator and data loader to the dependency injection container.
+    builder.Services.AddSingleton<UniqueKeyGenerator<Guid>>(new UniqueKeyGenerator<Guid>(() => Guid.NewGuid()));
+    builder.Services.AddSingleton<UniqueKeyGenerator<string>>(new UniqueKeyGenerator<string>(() => Guid.NewGuid().ToString()));
+    builder.Services.AddSingleton<IDataLoader, DataLoader<TKey>>();
+
+    // Add the main service for this application.
+    builder.Services.AddHostedService<RAGChatService<TKey>>();
+}

dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs

@@ -0,0 +1,187 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Data;
+using Microsoft.SemanticKernel.PromptTemplates.Handlebars;
+using VectorStoreRAG.Options;
+
+namespace VectorStoreRAG;
+
+/// <summary>
+/// Main service class for the application.
+/// </summary>
+/// <typeparam name="TKey">The type of the data model key.</typeparam>
+/// <param name="dataLoader">Used to load data into the vector store.</param>
+/// <param name="vectorStoreTextSearch">Used to search the vector store.</param>
+/// <param name="kernel">Used to make requests to the LLM.</param>
+/// <param name="ragConfigOptions">The configuration options for the application.</param>
+/// <param name="appShutdownCancellationTokenSource">Used to gracefully shut down the entire application when cancelled.</param>
+internal sealed class RAGChatService<TKey>(
+    IDataLoader dataLoader,
+    VectorStoreTextSearch<TextSnippet<TKey>> vectorStoreTextSearch,
+    Kernel kernel,
+    IOptions<RagConfig> ragConfigOptions,
+    [FromKeyedServices("AppShutdown")] CancellationTokenSource appShutdownCancellationTokenSource) : IHostedService
+{
+    private Task? _dataLoaded;
+    private Task? _chatLoop;
+
+    /// <summary>
+    /// Start the service.
+    /// </summary>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>An async task that completes when the service is started.</returns>
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        // Start to load all the configured PDFs into the vector store.
+        if (ragConfigOptions.Value.BuildCollection)
+        {
+            this._dataLoaded = this.LoadDataAsync(cancellationToken);
+        }
+        else
+        {
+            this._dataLoaded = Task.CompletedTask;
+        }
+
+        // Start the chat loop.
+        this._chatLoop = this.ChatLoopAsync(cancellationToken);
+
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Stop the service.
+    /// </summary>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>An async task that completes when the service is stopped.</returns>
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Contains the main chat loop for the application.
+    /// </summary>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>An async task that completes when the chat loop is shut down.</returns>
+    private async Task ChatLoopAsync(CancellationToken cancellationToken)
+    {
+        var pdfFiles = string.Join(", ", ragConfigOptions.Value.PdfFilePaths ?? []);
+
+        // Wait for the data to be loaded before starting the chat loop.
+        while (this._dataLoaded != null && !this._dataLoaded.IsCompleted && !cancellationToken.IsCancellationRequested)
+        {
+            await Task.Delay(1_000, cancellationToken).ConfigureAwait(false);
+        }
+
+        // If data loading failed, don't start the chat loop.
+        if (this._dataLoaded != null && this._dataLoaded.IsFaulted)
+        {
+            Console.WriteLine("Failed to load data");
+            return;
+        }
+
+        Console.WriteLine("PDF loading complete\n");
+
+        Console.ForegroundColor = ConsoleColor.Green;
+        Console.WriteLine("Assistant > Press enter with no prompt to exit.");
+
+        // Add a search plugin to the kernel which we will use in the template below
+        // to do a vector search for related information to the user query.
+        kernel.Plugins.Add(vectorStoreTextSearch.CreateWithGetTextSearchResults("SearchPlugin"));
+
+        // Start the chat loop.
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            // Prompt the user for a question.
+            Console.ForegroundColor = ConsoleColor.Green;
+            Console.WriteLine($"Assistant > What would you like to know from the loaded PDFs: ({pdfFiles})?");
+
+            // Read the user question.
+            Console.ForegroundColor = ConsoleColor.White;
+            Console.Write("User > ");
+            var question = Console.ReadLine();
+
+            // Exit the application if the user didn't type anything.
+            if (string.IsNullOrWhiteSpace(question))
+            {
+                appShutdownCancellationTokenSource.Cancel();
+                break;
+            }
+
+            // Invoke the LLM with a template that uses the search plugin to
+            // 1. get related information to the user query from the vector store
+            // 2. add the information to the LLM prompt.
+            var response = kernel.InvokePromptStreamingAsync(
+                promptTemplate: """
+                    Please use this information to answer the question:
+                    {{#with (SearchPlugin-GetTextSearchResults question)}}  
+                      {{#each this}}  
+                        Name: {{Name}}
+                        Value: {{Value}}
+                        Link: {{Link}}
+                        -----------------
+                      {{/each}}
+                    {{/with}}
+
+                    Include citations to the relevant information where it is referenced in the response.
+                    
+                    Question: {{question}}
+                    """,
+                arguments: new KernelArguments()
+                {
+                    { "question", question },
+                },
+                templateFormat: "handlebars",
+                promptTemplateFactory: new HandlebarsPromptTemplateFactory(),
+                cancellationToken: cancellationToken);
+
+            // Stream the LLM response to the console with error handling.
+            Console.ForegroundColor = ConsoleColor.Green;
+            Console.Write("\nAssistant > ");
+
+            try
+            {
+                await foreach (var message in response.ConfigureAwait(false))
+                {
+                    Console.Write(message);
+                }
+                Console.WriteLine();
+            }
+            catch (Exception ex)
+            {
+                Console.ForegroundColor = ConsoleColor.Red;
+                Console.WriteLine($"Call to LLM failed with error: {ex}");
+            }
+        }
+    }
+
+    /// <summary>
+    /// Load all configured PDFs into the vector store.
+    /// </summary>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>An async task that completes when the loading is complete.</returns>
+    private async Task LoadDataAsync(CancellationToken cancellationToken)
+    {
+        try
+        {
+            foreach (var pdfFilePath in ragConfigOptions.Value.PdfFilePaths ?? [])
+            {
+                Console.WriteLine($"Loading PDF into vector store: {pdfFilePath}");
+                await dataLoader.LoadPdf(
+                    pdfFilePath,
+                    ragConfigOptions.Value.DataLoadingBatchSize,
+                    ragConfigOptions.Value.DataLoadingBetweenBatchDelayInMilliseconds,
+                    cancellationToken).ConfigureAwait(false);
+            }
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Failed to load PDFs: {ex}");
+            throw;
+        }
+    }
+}

dotnet/samples/Demos/VectorStoreRAG/README.md

@@ -0,0 +1,182 @@
+# Vector Store RAG Demo
+
+This sample demonstrates how to ingest text from pdf files into a vector store and ask questions about the content
+using an LLM while using RAG to supplement the LLM with additional information from the vector store.
+
+## Configuring the Sample
+
+The sample can be configured in various ways:
+
+1. You can choose your preferred vector store by setting the `Rag:VectorStoreType` configuration setting in the `appsettings.json` file to one of the following values:
+   1. AzureAISearch
+   1. CosmosMongoDB
+   1. CosmosNoSql
+   1. InMemory
+   1. Qdrant
+   1. Redis
+   1. Weaviate
+1. You can choose your preferred AI Chat service by settings the `Rag:AIChatService` configuration setting in the `appsettings.json` file to one of the following values:
+   1. AzureOpenAI
+   1. OpenAI
+1. You can choose your preferred AI Embedding service by settings the `Rag:AIEmbeddingService` configuration setting in the `appsettings.json` file to one of the following values:
+   1. AzureOpenAIEmbeddings
+   1. OpenAIEmbeddings
+1. You can choose whether to load data into the vector store by setting the `Rag:BuildCollection` configuration setting in the `appsettings.json` file to `true`. If you set this to `false`, the sample will assume that data was already loaded previously and it will go straight into the chat experience.
+1. You can choose the name of the collection to use by setting the `Rag:CollectionName` configuration setting in the `appsettings.json` file.
+1. You can choose the pdf file to load into the vector store by setting the `Rag:PdfFilePaths` array in the `appsettings.json` file.
+1. You can choose the number of records to process per batch when loading data into the vector store by setting the `Rag:DataLoadingBatchSize` configuration setting in the `appsettings.json` file.
+1. You can choose the number of milliseconds to wait between batches when loading data into the vector store by setting the `Rag:DataLoadingBetweenBatchDelayInMilliseconds` configuration setting in the `appsettings.json` file.
+
+## Dependency Setup
+
+To run this sample, you need to setup your source data, setup your vector store and AI services, and setup secrets for these.
+
+### Source PDF File
+
+You will need to supply some source pdf files to load into the vector store.
+Once you have a file ready, update the `PdfFilePaths` array in the `appsettings.json` file with the path to the file.
+
+```json
+{
+    "Rag": {
+        "PdfFilePaths": [ "sourcedocument.pdf" ],
+    }
+}
+```
+
+Why not try the semantic kernel documentation as your input.
+You can download it as a PDF from the https://learn.microsoft.com/en-us/semantic-kernel/overview/ page.
+See the Download PDF button at the bottom of the page.
+
+### Azure OpenAI Chat Completion
+
+For Azure OpenAI Chat Completion, you need to add the following secrets:
+
+```cli
+dotnet user-secrets set "AIServices:AzureOpenAI:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAI:ChatDeploymentName" "<your deployment name>"
+```
+
+Note that the code doesn't use an API Key to communicate with Azure OpenAI, but rather an `AzureCliCredential` so no api key secret is required.
+
+### OpenAI Chat Completion
+
+For OpenAI Chat Completion, you need to add the following secrets:
+
+```cli
+dotnet user-secrets set "AIServices:OpenAI:ModelId" "<your model id>"
+dotnet user-secrets set "AIServices:OpenAI:ApiKey" "<your api key>"
+```
+
+Optionally, you can also provide an Org Id
+
+```cli
+dotnet user-secrets set "AIServices:OpenAI:OrgId" "<your org id>"
+```
+
+### Azure OpenAI Embeddings
+
+For Azure OpenAI Embeddings, you need to add the following secrets:
+
+```cli
+dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:DeploymentName" "<your deployment name>"
+```
+
+Note that the code doesn't use an API Key to communicate with Azure OpenAI, but rather an `AzureCliCredential` so no api key secret is required.
+
+### OpenAI Embeddings
+
+For OpenAI Embeddings, you need to add the following secrets:
+
+```cli
+dotnet user-secrets set "AIServices:OpenAIEmbeddings:ModelId" "<your model id>"
+dotnet user-secrets set "AIServices:OpenAIEmbeddings:ApiKey" "<your api key>"
+```
+
+Optionally, you can also provide an Org Id
+
+```cli
+dotnet user-secrets set "AIServices:OpenAIEmbeddings:OrgId" "<your org id>"
+```
+
+### Azure AI Search
+
+If you want to use Azure AI Search as your vector store, you will need to create an instance of Azure AI Search and add
+the following secrets here:
+
+```cli
+dotnet user-secrets set "VectorStores:AzureAISearch:Endpoint" "https://<yourservice>.search.windows.net"
+dotnet user-secrets set "VectorStores:AzureAISearch:ApiKey" "<yoursecret>"
+```
+
+### Azure CosmosDB MongoDB
+
+If you want to use Azure CosmosDB MongoDB as your vector store, you will need to create an instance of Azure CosmosDB MongoDB and add
+the following secrets here:
+
+```cli
+dotnet user-secrets set "VectorStores:CosmosMongoDB:ConnectionString" "<yourconnectionstring>"
+dotnet user-secrets set "VectorStores:CosmosMongoDB:DatabaseName" "<yourdbname>"
+```
+
+### Azure CosmosDB NoSQL
+
+If you want to use Azure CosmosDB NoSQL as your vector store, you will need to create an instance of Azure CosmosDB NoSQL and add
+the following secrets here:
+
+```cli
+dotnet user-secrets set "VectorStores:CosmosNoSql:ConnectionString" "<yourconnectionstring>"
+dotnet user-secrets set "VectorStores:CosmosNoSql:DatabaseName" "<yourdbname>"
+```
+
+### Qdrant
+
+If you want to use Qdrant as your vector store, you will need to have an instance of Qdrant available.
+
+You can use the following command to start a Qdrant instance in docker, and this will work with the default configured settings:
+
+```cli
+docker run -d --name qdrant -p 6333:6333 -p 6334:6334 qdrant/qdrant:latest
+```
+
+If you want to use a different instance of Qdrant, you can update the appsettings.json file or add the following secrets to reconfigure:
+
+```cli
+dotnet user-secrets set "VectorStores:Qdrant:Host" "<yourservice>"
+dotnet user-secrets set "VectorStores:Qdrant:Port" "6334"
+dotnet user-secrets set "VectorStores:Qdrant:Https" "true"
+dotnet user-secrets set "VectorStores:Qdrant:ApiKey" "<yoursecret>"
+```
+
+### Redis
+
+If you want to use Redis as your vector store, you will need to have an instance of Redis available.
+
+You can use the following command to start a Redis instance in docker, and this will work with the default configured settings:
+
+```cli
+docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+```
+
+If you want to use a different instance of Redis, you can update the appsettings.json file or add the following secret to reconfigure:
+
+```cli
+dotnet user-secrets set "VectorStores:Redis:ConnectionConfiguration" "<yourredisconnectionconfiguration>"
+```
+
+### Weaviate
+
+If you want to use Weaviate as your vector store, you will need to have an instance of Weaviate available.
+
+You can use the following command to start a Weaviate instance in docker, and this will work with the default configured settings:
+
+```cli
+docker run -d --name weaviate -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/weaviate:1.26.4
+```
+
+If you want to use a different instance of Weaviate, you can update the appsettings.json file or add the following secret to reconfigure:
+
+```cli
+dotnet user-secrets set "VectorStores:Weaviate:Endpoint" "<yourweaviateurl>"
+```

dotnet/samples/Demos/VectorStoreRAG/TextSnippet.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Data;
+
+namespace VectorStoreRAG;
+
+/// <summary>
+/// Data model for storing a section of text with an embedding and an optional reference link.
+/// </summary>
+/// <typeparam name="TKey">The type of the data model key.</typeparam>
+internal sealed class TextSnippet<TKey>
+{
+    [VectorStoreKey]
+    public required TKey Key { get; set; }
+
+    [TextSearchResultValue]
+    [VectorStoreData]
+    public string? Text { get; set; }
+
+    [TextSearchResultName]
+    [VectorStoreData]
+    public string? ReferenceDescription { get; set; }
+
+    [TextSearchResultLink]
+    [VectorStoreData]
+    public string? ReferenceLink { get; set; }
+
+    /// <summary>
+    /// The text embedding for this snippet. This is used to search the vector store.
+    /// While this is a string property it has the vector attribute, which means whatever
+    /// text it contains will be converted to a vector and stored as a vector in the vector store.
+    /// </summary>
+    [VectorStoreVector(1536)]
+    public string? TextEmbedding => this.Text;
+}

dotnet/samples/Demos/VectorStoreRAG/UniqueKeyGenerator.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VectorStoreRAG;
+
+/// <summary>
+/// Class for generating unique keys via a provided function.
+/// </summary>
+/// <typeparam name="TKey">The type of key to generate.</typeparam>
+/// <param name="generator">The function to generate the key with.</param>
+internal sealed class UniqueKeyGenerator<TKey>(Func<TKey> generator)
+    where TKey : notnull
+{
+    /// <summary>
+    /// Generate a unique key.
+    /// </summary>
+    /// <returns>The unique key that was generated.</returns>
+    public TKey GenerateKey() => generator();
+}

dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <NoWarn>$(NoWarn);SKEXP0001;SKEXP0010</NoWarn>
+    <UserSecretsId>c4203b00-7179-47c1-8701-ee352e381412</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" />
+    <PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" />
+    <PackageReference Include="PdfPig" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.CosmosMongoDB" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.CosmosNoSql" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Redis" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Weaviate" />
+    <PackageReference Include="SharpCompress" /> <!-- Pin to patched version; overrides transitive 0.30.1 from MongoDB.Driver (GHSA-6c8g-7p36-r338) -->
+    <PackageReference Include="Snappier" /> <!-- Pin to patched version; overrides transitive 1.0.0 from MongoDB.Driver (GHSA-pggp-6c3x-2xmx) -->
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
+    <ProjectReference Include="..\..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
+    <ProjectReference Include="..\..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

dotnet/samples/Demos/VectorStoreRAG/appsettings.json

@@ -0,0 +1,63 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "None"
+    }
+  },
+  "AIServices": {
+    "AzureOpenAI": {
+      "Endpoint": "",
+      "ChatDeploymentName": "gpt-4o"
+    },
+    "AzureOpenAIEmbeddings": {
+      "Endpoint": "",
+      "DeploymentName": "text-embedding-ada-002"
+    },
+    "OpenAI": {
+      "ModelId": "gpt-4o",
+      "ApiKey": "",
+      "OrgId": null
+    },
+    "OpenAIEmbeddings": {
+      "ModelId": "text-embedding-3-small",
+      "ApiKey": "",
+      "OrgId": null
+    }
+  },
+  "VectorStores": {
+    "AzureAISearch": {
+      "Endpoint": "",
+      "ApiKey": ""
+    },
+    "CosmosMongoDB": {
+      "ConnectionString": "",
+      "DatabaseName": ""
+    },
+    "CosmosNoSql": {
+      "ConnectionString": "",
+      "DatabaseName": ""
+    },
+    "Qdrant": {
+      "Host": "localhost",
+      "Port": 6334,
+      "Https": false,
+      "ApiKey": ""
+    },
+    "Redis": {
+      "ConnectionConfiguration": "localhost:6379"
+    },
+    "Weaviate": {
+      "Endpoint": "http://localhost:8080/v1/"
+    }
+  },
+  "Rag": {
+    "AIChatService": "OpenAI",
+    "AIEmbeddingService": "OpenAIEmbeddings",
+    "BuildCollection": true,
+    "CollectionName": "pdfcontent",
+    "DataLoadingBatchSize": 10,
+    "DataLoadingBetweenBatchDelayInMilliseconds": 1000,
+    "PdfFilePaths": [ "sourcedocument.pdf" ],
+    "VectorStoreType": "InMemory"
+  }
+}