chore: import upstream snapshot with attribution

dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj

@@ -0,0 +1,51 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <AssemblyName>GettingStartedWithVectorStores</AssemblyName>
+    <RootNamespace></RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <TargetFramework>net10.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <IsPackable>false</IsPackable>
+    <!-- Suppress: "Declare types in namespaces", "Require ConfigureAwait", "Experimental" -->
+    <NoWarn>$(NoWarn);CS8618,IDE0009,IDE1006,CA1051,CA1050,CA1707,CA1054,CA2007,VSTHRD111,CS1591,RCS1110,RCS1243,CA5394,SKEXP0001,SKEXP0010,SKEXP0040,SKEXP0050,SKEXP0060,SKEXP0101</NoWarn>
+    <OutputType>Library</OutputType>
+    <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xRetry" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.abstractions" />
+    <PackageReference Include="xunit.runner.visualstudio">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+    <PackageReference Include="Microsoft.Extensions.Http" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" />
+    <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
+    <PackageReference Include="Microsoft.Extensions.Logging" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.AzureAISearch" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.Redis" />
+  </ItemGroup>
+  
+  <Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/samples/SamplesInternalUtilities.props" />
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Connectors\Connectors.AzureOpenAI\Connectors.AzureOpenAI.csproj" />
+    <ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
+    <ProjectReference Include="..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <Using Include="Xunit" />
+    <Using Include="Xunit.Abstractions" />
+  </ItemGroup>
+</Project>

dotnet/samples/GettingStartedWithVectorStores/Glossary.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Sample model class that represents a glossary entry.
+/// </summary>
+/// <remarks>
+/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
+/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
+/// </remarks>
+internal sealed class Glossary
+{
+    [VectorStoreKey]
+    public string Key { get; set; }
+
+    [VectorStoreData(IsIndexed = true)]
+    public string Category { get; set; }
+
+    [VectorStoreData]
+    public string Term { get; set; }
+
+    [VectorStoreData]
+    public string Definition { get; set; }
+
+    [VectorStoreVector(Dimensions: 1536)]
+    public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
+}

dotnet/samples/GettingStartedWithVectorStores/README.md

@@ -0,0 +1,37 @@
+# Starting With Semantic Kernel Vector Stores
+
+This project contains a step by step guide to get started using Vector Stores with the Semantic Kernel.
+
+The examples can be run as integration tests but their code can also be copied to stand-alone programs.
+
+## Configuring Secrets
+
+Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
+Vector Stores and other resources. We suggest using .NET
+[Secret Manager](https://learn.microsoft.com/aspnet/core/security/app-secrets)
+to avoid the risk of leaking secrets into the repository, branches and pull requests.
+You can also use environment variables if you prefer.
+
+To set your secrets with Secret Manager:
+
+```
+cd dotnet/samples/GettingStartedWithVectorStores
+
+dotnet user-secrets init
+
+dotnet user-secrets set "AzureOpenAIEmbeddings:DeploymentName" "..."
+dotnet user-secrets set "AzureOpenAIEmbeddings:Endpoint" "..."
+
+dotnet user-secrets set "AzureAISearch:Endpoint" "..."
+dotnet user-secrets set "AzureAISearch:ApiKey" "..."
+```
+
+To set your secrets with environment variables, use these names:
+
+```
+AzureOpenAIEmbeddings__DeploymentName
+AzureOpenAIEmbeddings__Endpoint
+
+AzureAISearch__Endpoint
+AzureAISearch__ApiKey
+```

dotnet/samples/GettingStartedWithVectorStores/Step1_Ingest_Data.cs

@@ -0,0 +1,113 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.InMemory;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Example showing how to generate embeddings and ingest data into an in-memory vector store.
+/// </summary>
+public class Step1_Ingest_Data(ITestOutputHelper output, VectorStoresFixture fixture) : BaseTest(output), IClassFixture<VectorStoresFixture>
+{
+    /// <summary>
+    /// Example showing how to ingest data into an in-memory vector store.
+    /// </summary>
+    [Fact]
+    public async Task IngestDataIntoInMemoryVectorStoreAsync()
+    {
+        // Construct the vector store and get the collection.
+        var vectorStore = new InMemoryVectorStore();
+        var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
+
+        // Ingest data into the collection.
+        await IngestDataIntoVectorStoreAsync(collection, fixture.EmbeddingGenerator);
+
+        // Retrieve an item from the collection and write it to the console.
+        var record = await collection.GetAsync("4");
+        Console.WriteLine(record!.Definition);
+    }
+
+    /// <summary>
+    /// Ingest data into the given collection.
+    /// </summary>
+    /// <param name="collection">The collection to ingest data into.</param>
+    /// <param name="embeddingGenerator">The service to use for generating embeddings.</param>
+    /// <returns>The keys of the upserted records.</returns>
+    internal static async Task<IEnumerable<string>> IngestDataIntoVectorStoreAsync(
+        VectorStoreCollection<string, Glossary> collection,
+        IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator)
+    {
+        // Create the collection if it doesn't exist.
+        await collection.EnsureCollectionExistsAsync();
+
+        // Create glossary entries and generate embeddings for them.
+        var glossaryEntries = CreateGlossaryEntries().ToList();
+        var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
+        {
+            entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
+        }));
+        await Task.WhenAll(tasks);
+
+        // Upsert the glossary entries into the collection and return their keys.
+        await collection.UpsertAsync(glossaryEntries);
+
+        return glossaryEntries.Select(g => g.Key);
+    }
+
+    /// <summary>
+    /// Create some sample glossary entries.
+    /// </summary>
+    /// <returns>A list of sample glossary entries.</returns>
+    private static IEnumerable<Glossary> CreateGlossaryEntries()
+    {
+        yield return new Glossary
+        {
+            Key = "1",
+            Category = "Software",
+            Term = "API",
+            Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
+        };
+
+        yield return new Glossary
+        {
+            Key = "2",
+            Category = "Software",
+            Term = "SDK",
+            Definition = "Software development kit. A set of libraries and tools that allow software developers to build software more easily."
+        };
+
+        yield return new Glossary
+        {
+            Key = "3",
+            Category = "SK",
+            Term = "Connectors",
+            Definition = "Semantic Kernel Connectors allow software developers to integrate with various services providing AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
+        };
+
+        yield return new Glossary
+        {
+            Key = "4",
+            Category = "SK",
+            Term = "Semantic Kernel",
+            Definition = "Semantic Kernel is a set of libraries that allow software developers to more easily develop applications that make use of AI experiences."
+        };
+
+        yield return new Glossary
+        {
+            Key = "5",
+            Category = "AI",
+            Term = "RAG",
+            Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
+        };
+
+        yield return new Glossary
+        {
+            Key = "6",
+            Category = "AI",
+            Term = "LLM",
+            Definition = "Large language model. A type of artificial intelligence algorithm that is designed to understand and generate human language."
+        };
+    }
+}

dotnet/samples/GettingStartedWithVectorStores/Step2_Vector_Search.cs

@@ -0,0 +1,89 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.InMemory;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Example showing how to do vector searches with an in-memory vector store.
+/// </summary>
+public class Step2_Vector_Search(ITestOutputHelper output, VectorStoresFixture fixture) : BaseTest(output), IClassFixture<VectorStoresFixture>
+{
+    /// <summary>
+    /// Do a basic vector search where we just want to retrieve the single most relevant result.
+    /// </summary>
+    [Fact]
+    public async Task SearchAnInMemoryVectorStoreAsync()
+    {
+        var collection = await GetVectorStoreCollectionWithDataAsync();
+
+        // Search the vector store.
+        var searchResultItem = await SearchVectorStoreAsync(
+            collection,
+            "What is an Application Programming Interface?",
+            fixture.EmbeddingGenerator);
+
+        // Write the search result with its score to the console.
+        Console.WriteLine(searchResultItem.Record.Definition);
+        Console.WriteLine(searchResultItem.Score);
+    }
+
+    /// <summary>
+    /// Search the given collection for the most relevant result to the given search string.
+    /// </summary>
+    /// <param name="collection">The collection to search.</param>
+    /// <param name="searchString">The string to search matches for.</param>
+    /// <param name="embeddingGenerator">The service to generate embeddings with.</param>
+    /// <returns>The top search result.</returns>
+    internal static async Task<VectorSearchResult<Glossary>> SearchVectorStoreAsync(VectorStoreCollection<string, Glossary> collection, string searchString, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator)
+    {
+        // Generate an embedding from the search string.
+        var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
+
+        // Search the store and get the single most relevant result.
+        var searchResultItems = await collection.SearchAsync(
+            searchVector,
+            top: 1).ToListAsync();
+        return searchResultItems.First();
+    }
+
+    /// <summary>
+    /// Do a more complex vector search with pre-filtering.
+    /// </summary>
+    [Fact]
+    public async Task SearchAnInMemoryVectorStoreWithFilteringAsync()
+    {
+        var collection = await GetVectorStoreCollectionWithDataAsync();
+
+        // Generate an embedding from the search string.
+        var searchString = "How do I provide additional context to an LLM?";
+        var searchVector = (await fixture.EmbeddingGenerator.GenerateAsync(searchString)).Vector;
+
+        // Search the store with a filter and get the single most relevant result.
+        var searchResultItems = await collection.SearchAsync(
+            searchVector,
+            top: 1,
+            new()
+            {
+                Filter = g => g.Category == "AI"
+            }).ToListAsync();
+
+        // Write the search result with its score to the console.
+        Console.WriteLine(searchResultItems.First().Record.Definition);
+        Console.WriteLine(searchResultItems.First().Score);
+    }
+
+    private async Task<VectorStoreCollection<string, Glossary>> GetVectorStoreCollectionWithDataAsync()
+    {
+        // Construct the vector store and get the collection.
+        var vectorStore = new InMemoryVectorStore();
+        var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
+
+        // Ingest data into the collection using the code from step 1.
+        await Step1_Ingest_Data.IngestDataIntoVectorStoreAsync(collection, fixture.EmbeddingGenerator);
+
+        return collection;
+    }
+}

dotnet/samples/GettingStartedWithVectorStores/Step3_Switch_VectorStore.cs

@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Azure;
+using Azure.Search.Documents.Indexes;
+using Microsoft.SemanticKernel.Connectors.AzureAISearch;
+using Microsoft.SemanticKernel.Connectors.Redis;
+using StackExchange.Redis;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Example that shows that you can switch between different vector stores with the same code.
+/// </summary>
+public class Step3_Switch_VectorStore(ITestOutputHelper output, VectorStoresFixture fixture) : BaseTest(output), IClassFixture<VectorStoresFixture>
+{
+    /// <summary>
+    /// Here we are going to use the same code that we used in <see cref="Step1_Ingest_Data"/> and <see cref="Step2_Vector_Search"/>
+    /// but now with an <see cref="AzureAISearchVectorStore"/>
+    ///
+    /// This example requires an Azure AI Search service to be available.
+    /// </summary>
+    [Fact]
+    public async Task UseAnAzureAISearchVectorStoreAsync()
+    {
+        // Construct an Azure AI Search vector store and get the collection.
+        var vectorStore = new AzureAISearchVectorStore(new SearchIndexClient(
+            new Uri(TestConfiguration.AzureAISearch.Endpoint),
+            new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey)));
+        var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
+
+        // Ingest data into the collection using the same code as we used in Step1 with the InMemory Vector Store.
+        await Step1_Ingest_Data.IngestDataIntoVectorStoreAsync(collection, fixture.EmbeddingGenerator);
+
+        // Search the vector store using the same code as we used in Step2 with the InMemory Vector Store.
+        var searchResultItem = await Step2_Vector_Search.SearchVectorStoreAsync(
+            collection,
+            "What is an Application Programming Interface?",
+            fixture.EmbeddingGenerator);
+
+        // Write the search result with its score to the console.
+        Console.WriteLine(searchResultItem.Record.Definition);
+        Console.WriteLine(searchResultItem.Score);
+    }
+
+    /// <summary>
+    /// Here we are going to use the same code that we used in <see cref="Step1_Ingest_Data"/> and <see cref="Step2_Vector_Search"/>
+    /// but now with a <see cref="RedisVectorStore"/>
+    ///
+    /// This example requires a Redis server running on localhost:6379. To run a Redis server in a Docker container, use the following command:
+    /// docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+    /// </summary>
+    [Fact]
+    public async Task UseARedisVectorStoreAsync()
+    {
+        // Construct a Redis vector store and get the collection.
+        var vectorStore = new RedisVectorStore(ConnectionMultiplexer.Connect("localhost:6379").GetDatabase());
+        var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
+
+        // Ingest data into the collection using the same code as we used in Step1 with the InMemory Vector Store.
+        await Step1_Ingest_Data.IngestDataIntoVectorStoreAsync(collection, fixture.EmbeddingGenerator);
+
+        // Search the vector store using the same code as we used in Step2 with the InMemory Vector Store.
+        var searchResultItem = await Step2_Vector_Search.SearchVectorStoreAsync(
+            collection,
+            "What is an Application Programming Interface?",
+            fixture.EmbeddingGenerator);
+
+        // Write the search result with its score to the console.
+        Console.WriteLine(searchResultItem.Record.Definition);
+        Console.WriteLine(searchResultItem.Score);
+    }
+}

dotnet/samples/GettingStartedWithVectorStores/Step4_Use_DynamicDataModel.cs

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Redis;
+using StackExchange.Redis;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Example that shows that you can use the dynamic data modeling to interact with a vector database.
+/// This makes it possible to use the vector store abstractions without having to create your own strongly-typed data model.
+/// </summary>
+public class Step4_Use_DynamicDataModel(ITestOutputHelper output, VectorStoresFixture fixture) : BaseTest(output), IClassFixture<VectorStoresFixture>
+{
+    /// <summary>
+    /// Example showing how to query a vector store that uses dynamic data modeling.
+    ///
+    /// This example requires a Redis server running on localhost:6379. To run a Redis server in a Docker container, use the following command:
+    /// docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+    /// </summary>
+    [Fact]
+    public async Task SearchAVectorStoreWithDynamicMappingAsync()
+    {
+        // Construct a redis vector store.
+        var vectorStore = new RedisVectorStore(ConnectionMultiplexer.Connect("localhost:6379").GetDatabase());
+
+        // First, let's use the code from step 1 to ingest data into the vector store
+        // using the custom data model, simulating a scenario where someone else ingested
+        // the data into the database previously.
+        var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
+        var customDataModelCollection = vectorStore.GetCollection<string, Glossary>("skglossary");
+        await Step1_Ingest_Data.IngestDataIntoVectorStoreAsync(customDataModelCollection, fixture.EmbeddingGenerator);
+
+        // To use dynamic data modeling, we still have to describe the storage schema to the vector store
+        // using a record definition. The benefit over a custom data model is that this definition
+        // does not have to be known at compile time.
+        // E.g. it can be read from a configuration or retrieved from a service.
+        var recordDefinition = new VectorStoreCollectionDefinition
+        {
+            Properties =
+            [
+                new VectorStoreKeyProperty("Key", typeof(string)),
+                new VectorStoreDataProperty("Category", typeof(string)),
+                new VectorStoreDataProperty("Term", typeof(string)),
+                new VectorStoreDataProperty("Definition", typeof(string)),
+                new VectorStoreVectorProperty("DefinitionEmbedding", typeof(ReadOnlyMemory<float>), 1536),
+            ]
+        };
+
+        // Now, let's create a collection that uses a dynamic data model.
+        var dynamicDataModelCollection = vectorStore.GetDynamicCollection("skglossary", recordDefinition);
+
+        // Generate an embedding from the search string.
+        var searchString = "How do I provide additional context to an LLM?";
+        var searchVector = (await fixture.EmbeddingGenerator.GenerateAsync(searchString)).Vector;
+
+        // Search the generic data model collection and get the single most relevant result.
+        var searchResultItems = await dynamicDataModelCollection.SearchAsync(
+            searchVector,
+            top: 1).ToListAsync();
+
+        // Write the search result with its score to the console.
+        // Note that here we can loop through all the properties
+        // without knowing the schema, since the properties are
+        // stored as a dictionary of string keys and object values
+        // when using the dynamic data model.
+        foreach (var property in searchResultItems.First().Record)
+        {
+            Console.WriteLine($"{property.Key}: {property.Value}");
+        }
+        Console.WriteLine(searchResultItems.First().Score);
+    }
+}

dotnet/samples/GettingStartedWithVectorStores/VectorStoresFixture.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Reflection;
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Configuration;
+
+namespace GettingStartedWithVectorStores;
+
+/// <summary>
+/// Fixture containing common setup logic for the samples.
+/// </summary>
+public class VectorStoresFixture
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="VectorStoresFixture"/> class.
+    /// </summary>
+    public VectorStoresFixture()
+    {
+        IConfigurationRoot configRoot = new ConfigurationBuilder()
+            .AddJsonFile("appsettings.Development.json", true)
+            .AddEnvironmentVariables()
+            .AddUserSecrets(Assembly.GetExecutingAssembly())
+            .Build();
+        TestConfiguration.Initialize(configRoot);
+
+        this.EmbeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
+            .GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
+            .AsIEmbeddingGenerator(1536);
+    }
+
+    /// <summary>
+    /// Gets the text embedding generation service
+    /// </summary>
+    public IEmbeddingGenerator<string, Embedding<float>> EmbeddingGenerator { get; }
+}