chore: import upstream snapshot with attribution
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:21:23 +08:00
commit b957a53def
5423 changed files with 863745 additions and 0 deletions
@@ -0,0 +1,32 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using xRetry;
namespace Memory;
// The following example shows how to use Semantic Kernel with AWS Bedrock API for embedding generation,
// including the ability to specify custom dimensions.
public class AWSBedrock_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
/// <summary>
/// This test demonstrates how to use the AWS Bedrock API embedding generation.
/// </summary>
[RetryFact(typeof(HttpOperationException))]
public async Task GenerateEmbeddings()
{
IKernelBuilder kernelBuilder = Kernel.CreateBuilder()
.AddBedrockEmbeddingGenerator(modelId: TestConfiguration.Bedrock.EmbeddingModelId! ?? "amazon.titan-embed-text-v1");
Kernel kernel = kernelBuilder.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings with the default dimensions for the model
var embeddings = await embeddingGenerator.GenerateAsync(
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default for current model) for the provided text");
}
}
@@ -0,0 +1,120 @@
// Copyright (c) Microsoft. All rights reserved.
using Google.Apis.Auth.OAuth2;
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using xRetry;
namespace Memory;
// The following example shows how to use Semantic Kernel with Google AI and Google's Vertex AI for embedding generation,
// including the ability to specify custom dimensions.
public class Google_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
/// <summary>
/// This test demonstrates how to use the Google Vertex AI embedding generation service with default dimensions.
/// </summary>
/// <remarks>
/// Currently custom dimensions are not supported for Vertex AI.
/// </remarks>
[RetryFact(typeof(HttpOperationException))]
public async Task GenerateEmbeddingWithDefaultDimensionsUsingVertexAI()
{
string? bearerToken = null;
Assert.NotNull(TestConfiguration.VertexAI.EmbeddingModelId);
Assert.NotNull(TestConfiguration.VertexAI.ClientId);
Assert.NotNull(TestConfiguration.VertexAI.ClientSecret);
Assert.NotNull(TestConfiguration.VertexAI.Location);
Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
kernelBuilder.AddVertexAIEmbeddingGenerator(
modelId: TestConfiguration.VertexAI.EmbeddingModelId!,
bearerTokenProvider: GetBearerToken,
location: TestConfiguration.VertexAI.Location,
projectId: TestConfiguration.VertexAI.ProjectId);
Kernel kernel = kernelBuilder.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings with the default dimensions for the model
var embeddings = await embeddingGenerator.GenerateAsync(
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default) for the provided text");
// Uses Google.Apis.Auth.OAuth2 to get the bearer token
async ValueTask<string> GetBearerToken()
{
if (!string.IsNullOrEmpty(bearerToken))
{
return bearerToken;
}
var credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
new ClientSecrets
{
ClientId = TestConfiguration.VertexAI.ClientId,
ClientSecret = TestConfiguration.VertexAI.ClientSecret
},
["https://www.googleapis.com/auth/cloud-platform"],
"user",
CancellationToken.None);
var userCredential = await credential.WaitAsync(CancellationToken.None);
bearerToken = userCredential.Token.AccessToken;
return bearerToken;
}
}
[RetryFact(typeof(HttpOperationException))]
public async Task GenerateEmbeddingWithDefaultDimensionsUsingGoogleAI()
{
Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
kernelBuilder.AddGoogleAIEmbeddingGenerator(
modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
apiKey: TestConfiguration.GoogleAI.ApiKey);
Kernel kernel = kernelBuilder.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings with the default dimensions for the model
var embeddings = await embeddingGenerator.GenerateAsync(
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default) for the provided text");
}
[RetryFact(typeof(HttpOperationException))]
public async Task GenerateEmbeddingWithCustomDimensionsUsingGoogleAI()
{
Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
// Specify custom dimensions for the embeddings
const int CustomDimensions = 512;
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
kernelBuilder.AddGoogleAIEmbeddingGenerator(
modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
apiKey: TestConfiguration.GoogleAI.ApiKey,
dimensions: CustomDimensions);
Kernel kernel = kernelBuilder.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings with the specified custom dimensions
var embeddings = await embeddingGenerator.GenerateAsync(
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (custom: '{CustomDimensions}') for the provided text");
// Verify that we received embeddings with our requested dimensions
Assert.Equal(CustomDimensions, embeddings[0].Vector.Length);
}
}
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using xRetry;
#pragma warning disable format // Format item can be simplified
#pragma warning disable CA1861 // Avoid constant arrays as arguments
namespace Memory;
// The following example shows how to use Semantic Kernel with HuggingFace API.
public class HuggingFace_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
[RetryFact(typeof(HttpOperationException))]
public async Task RunInferenceApiEmbeddingAsync()
{
Console.WriteLine("\n======= Hugging Face Inference API - Embedding Example ========\n");
Kernel kernel = Kernel.CreateBuilder()
.AddHuggingFaceEmbeddingGenerator(
model: TestConfiguration.HuggingFace.EmbeddingModelId,
apiKey: TestConfiguration.HuggingFace.ApiKey)
.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings for each chunk.
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
}
}
@@ -0,0 +1,89 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.HuggingFace;
using Microsoft.SemanticKernel.Connectors.InMemory;
using Microsoft.SemanticKernel.Embeddings;
#pragma warning disable CS8602 // Dereference of a possibly null reference.
namespace Memory;
/// <summary>
/// This example shows how to use custom <see cref="HttpClientHandler"/> to override Hugging Face HTTP response.
/// Generally, an embedding model will return results as a 1 * n matrix for input type [string]. However, the model can have different matrix dimensionality.
/// For example, the <a href="https://huggingface.co/cointegrated/LaBSE-en-ru">cointegrated/LaBSE-en-ru</a> model returns results as a 1 * 1 * 4 * 768 matrix, which is different from Hugging Face embedding generation service implementation.
/// To address this, a custom <see cref="HttpClientHandler"/> can be used to modify the response before sending it back.
/// </summary>
[Obsolete("The IMemoryStore abstraction is being obsoleted")]
public class HuggingFace_TextEmbeddingCustomHttpHandler(ITestOutputHelper output) : BaseTest(output)
{
public async Task RunInferenceApiEmbeddingCustomHttpHandlerAsync()
{
Console.WriteLine("\n======= Hugging Face Inference API - Embedding Example ========\n");
var hf = new HuggingFaceTextEmbeddingGenerationService(
"cointegrated/LaBSE-en-ru",
apiKey: TestConfiguration.HuggingFace.ApiKey,
httpClient: new HttpClient(new CustomHttpClientHandler()
{
CheckCertificateRevocationList = true
})
);
var inMemoryCollection = new InMemoryCollection<string, Record>(
name: "Test",
new() { EmbeddingGenerator = hf.AsEmbeddingGenerator() });
await inMemoryCollection.UpsertAsync(new Record
{
Id = "1",
Text = "THIS IS A SAMPLE",
Embedding = "An embedding will be generated from this text"
});
}
public class Record
{
[VectorStoreKey]
public string Id { get; set; }
[VectorStoreData]
public string Text { get; set; }
[VectorStoreVector(Dimensions: 768)]
public string Embedding { get; set; }
}
private sealed class CustomHttpClientHandler : HttpClientHandler
{
private readonly JsonSerializerOptions _jsonOptions = new();
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
// Log the request URI
//Console.WriteLine($"Request: {request.Method} {request.RequestUri}");
// Send the request and get the response
HttpResponseMessage response = await base.SendAsync(request, cancellationToken);
// Log the response status code
//Console.WriteLine($"Response: {(int)response.StatusCode} {response.ReasonPhrase}");
// You can manipulate the response here
// For example, add a custom header
// response.Headers.Add("X-Custom-Header", "CustomValue");
// For example, modify the response content
Stream originalContent = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
List<List<List<ReadOnlyMemory<float>>>> modifiedContent = (await JsonSerializer.DeserializeAsync<List<List<List<ReadOnlyMemory<float>>>>>(originalContent, _jsonOptions, cancellationToken).ConfigureAwait(false))!;
Stream modifiedStream = new MemoryStream();
await JsonSerializer.SerializeAsync(modifiedStream, modifiedContent[0][0].ToList(), _jsonOptions, cancellationToken).ConfigureAwait(false);
response.Content = new StreamContent(modifiedStream);
// Return the modified response
return response;
}
}
}
@@ -0,0 +1,32 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using xRetry;
namespace Memory;
// The following example shows how to use Semantic Kernel with Ollama API.
public class Ollama_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
[RetryFact(typeof(HttpOperationException))]
public async Task RunEmbeddingAsync()
{
Assert.NotNull(TestConfiguration.Ollama.EmbeddingModelId);
Console.WriteLine("\n======= Ollama - Embedding Example ========\n");
Kernel kernel = Kernel.CreateBuilder()
.AddOllamaEmbeddingGenerator(
endpoint: new Uri(TestConfiguration.Ollama.Endpoint),
modelId: TestConfiguration.Ollama.EmbeddingModelId)
.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings for each chunk.
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
}
}
@@ -0,0 +1,82 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
namespace Memory;
// The following example shows how to use Semantic Kernel with Onnx GenAI API.
public class Onnx_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
/// <summary>
/// Example using the service directly to get embeddings
/// </summary>
/// <remarks>
/// Configuration example:
/// <list type="table">
/// <item>
/// <term>EmbeddingModelPath:</term>
/// <description>D:\huggingface\bge-micro-v2\onnx\model.onnx</description>
/// </item>
/// <item>
/// <term>EmbeddingVocabPath:</term>
/// <description>D:\huggingface\bge-micro-v2\vocab.txt</description>
/// </item>
/// </list>
/// </remarks>
[Fact]
public async Task RunEmbeddingAsync()
{
Assert.NotNull(TestConfiguration.Onnx.EmbeddingModelPath); // dotnet user-secrets set "Onnx:EmbeddingModelPath" "<model-file-path>"
Assert.NotNull(TestConfiguration.Onnx.EmbeddingVocabPath); // dotnet user-secrets set "Onnx:EmbeddingVocabPath" "<vocab-file-path>"
Console.WriteLine("\n======= Onnx - Embedding Example ========\n");
Kernel kernel = Kernel.CreateBuilder()
.AddBertOnnxEmbeddingGenerator(TestConfiguration.Onnx.EmbeddingModelPath, TestConfiguration.Onnx.EmbeddingVocabPath)
.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings for each chunk.
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
}
/// <summary>
/// Example using the service collection directly to get embeddings
/// </summary>
/// <remarks>
/// Configuration example:
/// <list type="table">
/// <item>
/// <term>EmbeddingModelPath:</term>
/// <description>D:\huggingface\bge-micro-v2\onnx\model.onnx</description>
/// </item>
/// <item>
/// <term>EmbeddingVocabPath:</term>
/// <description>D:\huggingface\bge-micro-v2\vocab.txt</description>
/// </item>
/// </list>
/// </remarks>
[Fact]
public async Task RunServiceCollectionEmbeddingAsync()
{
Assert.NotNull(TestConfiguration.Onnx.EmbeddingModelPath); // dotnet user-secrets set "Onnx:EmbeddingModelPath" "<model-file-path>"
Assert.NotNull(TestConfiguration.Onnx.EmbeddingVocabPath); // dotnet user-secrets set "Onnx:EmbeddingVocabPath" "<vocab-file-path>"
Console.WriteLine("\n======= Onnx - Embedding Example ========\n");
var services = new ServiceCollection()
.AddBertOnnxEmbeddingGenerator(TestConfiguration.Onnx.EmbeddingModelPath, TestConfiguration.Onnx.EmbeddingVocabPath);
var provider = services.BuildServiceProvider();
var embeddingGenerator = provider.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings for each chunk.
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
}
}
@@ -0,0 +1,34 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using xRetry;
#pragma warning disable format // Format item can be simplified
#pragma warning disable CA1861 // Avoid constant arrays as arguments
namespace Memory;
// The following example shows how to use Semantic Kernel with OpenAI.
public class OpenAI_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
{
[RetryFact(typeof(HttpOperationException))]
public async Task RunEmbeddingAsync()
{
Assert.NotNull(TestConfiguration.OpenAI.EmbeddingModelId);
Assert.NotNull(TestConfiguration.OpenAI.ApiKey);
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
kernelBuilder.AddOpenAIEmbeddingGenerator(
modelId: TestConfiguration.OpenAI.EmbeddingModelId!,
apiKey: TestConfiguration.OpenAI.ApiKey!);
Kernel kernel = kernelBuilder.Build();
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
// Generate embeddings for the specified text.
var embeddings = await embeddingGenerator.GenerateAsync(["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your C#, Python, or Java codebase."]);
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
}
}
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
using Microsoft.ML.Tokenizers;
using Microsoft.SemanticKernel.Text;
namespace Memory;
public class TextChunkerUsage(ITestOutputHelper output) : BaseTest(output)
{
private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
[Fact]
public void RunExample()
{
Console.WriteLine("=== Text chunking ===");
var lines = TextChunker.SplitPlainTextLines(Text, 40);
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120);
WriteParagraphsToConsole(paragraphs);
}
[Fact]
public void RunExampleWithTokenCounter()
{
Console.WriteLine("=== Text chunking with a custom token counter ===");
var sw = new Stopwatch();
sw.Start();
var lines = TextChunker.SplitPlainTextLines(Text, 40, text => s_tokenizer.CountTokens(text));
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120, tokenCounter: text => s_tokenizer.CountTokens(text));
sw.Stop();
Console.WriteLine($"Elapsed time: {sw.ElapsedMilliseconds} ms");
WriteParagraphsToConsole(paragraphs);
}
[Fact]
public void RunExampleWithHeader()
{
Console.WriteLine("=== Text chunking with chunk header ===");
var lines = TextChunker.SplitPlainTextLines(Text, 40);
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 150, chunkHeader: "DOCUMENT NAME: test.txt\n\n");
WriteParagraphsToConsole(paragraphs);
}
private void WriteParagraphsToConsole(List<string> paragraphs)
{
for (var i = 0; i < paragraphs.Count; i++)
{
Console.WriteLine(paragraphs[i]);
if (i < paragraphs.Count - 1)
{
Console.WriteLine("------------------------");
}
}
}
private const string Text = """
The city of Venice, located in the northeastern part of Italy,
is renowned for its unique geographical features. Built on more than 100 small islands in a lagoon in the
Adriatic Sea, it has no roads, just canals including the Grand Canal thoroughfare lined with Renaissance and
Gothic palaces. The central square, Piazza San Marco, contains St. Mark's Basilica, which is tiled with Byzantine
mosaics, and the Campanile bell tower offering views of the city's red roofs.
The Amazon Rainforest, also known as Amazonia, is a moist broadleaf tropical rainforest in the Amazon biome that
covers most of the Amazon basin of South America. This basin encompasses 7 million square kilometers, of which
5.5 million square kilometers are covered by the rainforest. This region includes territory belonging to nine nations
and 3.4 million square kilometers of uncontacted tribes. The Amazon represents over half of the planet's remaining
rainforests and comprises the largest and most biodiverse tract of tropical rainforest in the world.
The Great Barrier Reef is the world's largest coral reef system composed of over 2,900 individual reefs and 900 islands
stretching for over 2,300 kilometers over an area of approximately 344,400 square kilometers. The reef is located in the
Coral Sea, off the coast of Queensland, Australia. The Great Barrier Reef can be seen from outer space and is the world's
biggest single structure made by living organisms. This reef structure is composed of and built by billions of tiny organisms,
known as coral polyps.
""";
}
@@ -0,0 +1,167 @@
// Copyright (c) Microsoft. All rights reserved.
using System.ClientModel;
using Azure.AI.OpenAI;
using Microsoft.Extensions.AI;
using Microsoft.ML.Tokenizers;
using Microsoft.SemanticKernel.Text;
namespace Memory;
public class TextChunkingAndEmbedding(ITestOutputHelper output) : BaseTest(output)
{
private const string EmbeddingModelName = "text-embedding-ada-002";
private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel(EmbeddingModelName);
[Fact]
public async Task RunAsync()
{
Console.WriteLine("======== Text Embedding ========");
await RunExampleAsync();
}
private async Task RunExampleAsync()
{
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new ApiKeyCredential(TestConfiguration.AzureOpenAIEmbeddings.ApiKey))
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator();
// To demonstrate batching we'll create abnormally small partitions.
var lines = TextChunker.SplitPlainTextLines(ChatTranscript, maxTokensPerLine: 10);
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, maxTokensPerParagraph: 25);
Console.WriteLine($"Split transcript into {paragraphs.Count} paragraphs");
// Azure OpenAI currently supports input arrays up to 16 for text-embedding-ada-002 (Version 2).
// Both require the max input token limit per API request to remain under 8191 for this model.
var chunks = paragraphs
.ChunkByAggregate(
seed: 0,
aggregator: (tokenCount, paragraph) => tokenCount + s_tokenizer.CountTokens(paragraph),
predicate: (tokenCount, index) => tokenCount < 8191 && index < 16)
.ToList();
Console.WriteLine($"Consolidated paragraphs into {chunks.Count}");
// Generate embeddings for each chunk.
for (var i = 0; i < chunks.Count; i++)
{
var chunk = chunks[i];
var embeddings = await embeddingGenerator.GenerateAsync(chunk);
Console.WriteLine($"Generated {embeddings.Count} embeddings from chunk {i + 1}");
}
}
#region Transcript
private const string ChatTranscript =
@"
John: Hello, how are you?
Jane: I'm fine, thanks. How are you?
John: I'm doing well, writing some example code.
Jane: That's great! I'm writing some example code too.
John: What are you writing?
Jane: I'm writing a chatbot.
John: That's cool. I'm writing a chatbot too.
Jane: What language are you writing it in?
John: I'm writing it in C#.
Jane: I'm writing it in Python.
John: That's cool. I need to learn Python.
Jane: I need to learn C#.
John: Can I try out your chatbot?
Jane: Sure, here's the link.
John: Thanks!
Jane: You're welcome.
Jane: Look at this poem my chatbot wrote:
Jane: Roses are red
Jane: Violets are blue
Jane: I'm writing a chatbot
Jane: What about you?
John: That's cool. Let me see if mine will write a poem, too.
John: Here's a poem my chatbot wrote:
John: The singularity of the universe is a mystery.
John: The universe is a mystery.
John: The universe is a mystery.
John: The universe is a mystery.
John: Looks like I need to improve mine, oh well.
Jane: You might want to try using a different model.
Jane: I'm using the GPT-3 model.
John: I'm using the GPT-2 model. That makes sense.
John: Here is a new poem after updating the model.
John: The universe is a mystery.
John: The universe is a mystery.
John: The universe is a mystery.
John: Yikes, it's really stuck isn't it. Would you help me debug my code?
Jane: Sure, what's the problem?
John: I'm not sure. I think it's a bug in the code.
Jane: I'll take a look.
Jane: I think I found the problem.
Jane: It looks like you're not passing the right parameters to the model.
John: Thanks for the help!
Jane: I'm now writing a bot to summarize conversations. I want to make sure it works when the conversation is long.
John: So you need to keep talking with me to generate a long conversation?
Jane: Yes, that's right.
John: Ok, I'll keep talking. What should we talk about?
Jane: I don't know, what do you want to talk about?
John: I don't know, it's nice how CoPilot is doing most of the talking for us. But it definitely gets stuck sometimes.
Jane: I agree, it's nice that CoPilot is doing most of the talking for us.
Jane: But it definitely gets stuck sometimes.
John: Do you know how long it needs to be?
Jane: I think the max length is 1024 tokens. Which is approximately 1024*4= 4096 characters.
John: That's a lot of characters.
Jane: Yes, it is.
John: I'm not sure how much longer I can keep talking.
Jane: I think we're almost there. Let me check.
Jane: I have some bad news, we're only half way there.
John: Oh no, I'm not sure I can keep going. I'm getting tired.
Jane: I'm getting tired too.
John: Maybe there is a large piece of text we can use to generate a long conversation.
Jane: That's a good idea. Let me see if I can find one. Maybe Lorem Ipsum?
John: Yeah, that's a good idea.
Jane: I found a Lorem Ipsum generator.
Jane: Here's a 4096 character Lorem Ipsum text:
Jane: Lorem ipsum dolor sit amet, con
Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam
Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam
Jane: Darn, it's just repeating stuff now.
John: I think we're done.
Jane: We're not though! We need like 1500 more characters.
John: Oh Cananda, our home and native land.
Jane: True patriot love in all thy sons command.
John: With glowing hearts we see thee rise.
Jane: The True North strong and free.
John: From far and wide, O Canada, we stand on guard for thee.
Jane: God keep our land glorious and free.
John: O Canada, we stand on guard for thee.
Jane: O Canada, we stand on guard for thee.
Jane: That was fun, thank you. Let me check now.
Jane: I think we need about 600 more characters.
John: Oh say can you see?
Jane: By the dawn's early light.
John: What so proudly we hailed.
Jane: At the twilight's last gleaming.
John: Whose broad stripes and bright stars.
Jane: Through the perilous fight.
John: O'er the ramparts we watched.
Jane: Were so gallantly streaming.
John: And the rockets' red glare.
Jane: The bombs bursting in air.
John: Gave proof through the night.
Jane: That our flag was still there.
John: Oh say does that star-spangled banner yet wave.
Jane: O'er the land of the free.
John: And the home of the brave.
Jane: Are you a Seattle Kraken Fan?
John: Yes, I am. I love going to the games.
Jane: I'm a Seattle Kraken Fan too. Who is your favorite player?
John: I like watching all the players, but I think my favorite is Matty Beniers.
Jane: Yeah, he's a great player. I like watching him too. I also like watching Jaden Schwartz.
John: Adam Larsson is another good one. The big cat!
Jane: WE MADE IT! It's long enough. Thank you!
John: You're welcome. I'm glad we could help. Goodbye!
Jane: Goodbye!
";
#endregion
}
@@ -0,0 +1,100 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Data;
namespace Memory;
/// <summary>
/// Extension methods for <see cref="VectorStore"/> which allow:
/// 1. Creating an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings.
/// </summary>
internal static class VectorStoreExtensions
{
/// <summary>
/// Delegate to create a record from a string.
/// </summary>
/// <typeparam name="TKey">Type of the record key.</typeparam>
/// <typeparam name="TRecord">Type of the record.</typeparam>
internal delegate TRecord CreateRecordFromString<TKey, TRecord>(string text, ReadOnlyMemory<float> vector) where TKey : notnull;
/// <summary>
/// Delegate to create a record from a <see cref="TextSearchResult"/>.
/// </summary>
/// <typeparam name="TKey">Type of the record key.</typeparam>
/// <typeparam name="TRecord">Type of the record.</typeparam>
internal delegate TRecord CreateRecordFromTextSearchResult<TKey, TRecord>(TextSearchResult searchResult, ReadOnlyMemory<float> vector) where TKey : notnull;
/// <summary>
/// Create a <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings by:
/// 1. Getting an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/>
/// 2. Generating embeddings for each string.
/// 3. Creating a record with a valid key for each string and it's embedding.
/// 4. Insert the records into the collection.
/// </summary>
/// <param name="vectorStore">Instance of <see cref="VectorStore"/> used to created the collection.</param>
/// <param name="collectionName">The collection name.</param>
/// <param name="entries">A list of strings.</param>
/// <param name="embeddingGenerator">An embedding generator.</param>
/// <param name="createRecord">A delegate which can create a record with a valid key for each string and it's embedding.</param>
internal static async Task<VectorStoreCollection<TKey, TRecord>> CreateCollectionFromListAsync<TKey, TRecord>(
this VectorStore vectorStore,
string collectionName,
string[] entries,
IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
CreateRecordFromString<TKey, TRecord> createRecord)
where TKey : notnull
where TRecord : class
{
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<TKey, TRecord>(collectionName);
await collection.EnsureCollectionExistsAsync().ConfigureAwait(false);
// Create records and generate embeddings for them.
var tasks = entries.Select(entry => Task.Run(async () =>
{
var record = createRecord(entry, (await embeddingGenerator.GenerateAsync(entry).ConfigureAwait(false)).Vector);
await collection.UpsertAsync(record).ConfigureAwait(false);
}));
await Task.WhenAll(tasks).ConfigureAwait(false);
return collection;
}
/// <summary>
/// Create a <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings by:
/// 1. Getting an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/>
/// 2. Generating embeddings for each string.
/// 3. Creating a record with a valid key for each string and it's embedding.
/// 4. Insert the records into the collection.
/// </summary>
/// <param name="vectorStore">Instance of <see cref="VectorStore"/> used to created the collection.</param>
/// <param name="collectionName">The collection name.</param>
/// <param name="searchResults">A list of <see cref="TextSearchResult" />s.</param>
/// <param name="embeddingGenerator">An embedding generator service.</param>
/// <param name="createRecord">A delegate which can create a record with a valid key for each string and it's embedding.</param>
internal static async Task<VectorStoreCollection<TKey, TRecord>> CreateCollectionFromTextSearchResultsAsync<TKey, TRecord>(
this VectorStore vectorStore,
string collectionName,
IList<TextSearchResult> searchResults,
IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
CreateRecordFromTextSearchResult<TKey, TRecord> createRecord)
where TKey : notnull
where TRecord : class
{
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<TKey, TRecord>(collectionName);
await collection.EnsureCollectionExistsAsync().ConfigureAwait(false);
// Create records and generate embeddings for them.
var tasks = searchResults.Select(searchResult => Task.Run(async () =>
{
var record = createRecord(searchResult, (await embeddingGenerator.GenerateAsync(searchResult.Value!).ConfigureAwait(false)).Vector);
await collection.UpsertAsync(record).ConfigureAwait(false);
}));
await Task.WhenAll(tasks).ConfigureAwait(false);
return collection;
}
}
@@ -0,0 +1,153 @@
// Copyright (c) Microsoft. All rights reserved.
using Docker.DotNet;
using Docker.DotNet.Models;
namespace Memory.VectorStoreFixtures;
/// <summary>
/// Helper class that creates and deletes containers for the vector store examples.
/// </summary>
internal static class VectorStoreInfra
{
/// <summary>
/// Setup the postgres pgvector container by pulling the image and running it.
/// </summary>
/// <param name="client">The docker client to create the container with.</param>
/// <returns>The id of the container.</returns>
public static async Task<string> SetupPostgresContainerAsync(DockerClient client)
{
await client.Images.CreateImageAsync(
new ImagesCreateParameters
{
FromImage = "pgvector/pgvector",
Tag = "pg16",
},
null,
new Progress<JSONMessage>());
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
{
Image = "pgvector/pgvector:pg16",
HostConfig = new HostConfig()
{
PortBindings = new Dictionary<string, IList<PortBinding>>
{
{"5432", new List<PortBinding> {new() {HostPort = "5432" } }},
},
PublishAllPorts = true
},
ExposedPorts = new Dictionary<string, EmptyStruct>
{
{ "5432", default },
},
Env =
[
"POSTGRES_USER=postgres",
"POSTGRES_PASSWORD=example",
],
});
await client.Containers.StartContainerAsync(
container.ID,
new ContainerStartParameters());
return container.ID;
}
/// <summary>
/// Setup the qdrant container by pulling the image and running it.
/// </summary>
/// <param name="client">The docker client to create the container with.</param>
/// <returns>The id of the container.</returns>
public static async Task<string> SetupQdrantContainerAsync(DockerClient client)
{
await client.Images.CreateImageAsync(
new ImagesCreateParameters
{
FromImage = "qdrant/qdrant",
Tag = "latest",
},
null,
new Progress<JSONMessage>());
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
{
Image = "qdrant/qdrant",
HostConfig = new HostConfig()
{
PortBindings = new Dictionary<string, IList<PortBinding>>
{
{"6333", new List<PortBinding> {new() {HostPort = "6333" } }},
{"6334", new List<PortBinding> {new() {HostPort = "6334" } }}
},
PublishAllPorts = true
},
ExposedPorts = new Dictionary<string, EmptyStruct>
{
{ "6333", default },
{ "6334", default }
},
});
await client.Containers.StartContainerAsync(
container.ID,
new ContainerStartParameters());
return container.ID;
}
/// <summary>
/// Setup the redis container by pulling the image and running it.
/// </summary>
/// <param name="client">The docker client to create the container with.</param>
/// <returns>The id of the container.</returns>
public static async Task<string> SetupRedisContainerAsync(DockerClient client)
{
await client.Images.CreateImageAsync(
new ImagesCreateParameters
{
FromImage = "redis/redis-stack",
Tag = "latest",
},
null,
new Progress<JSONMessage>());
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
{
Image = "redis/redis-stack",
HostConfig = new HostConfig()
{
PortBindings = new Dictionary<string, IList<PortBinding>>
{
{"6379", new List<PortBinding> {new() {HostPort = "6379"}}},
{"8001", new List<PortBinding> {new() {HostPort = "8001"}}}
},
PublishAllPorts = true
},
ExposedPorts = new Dictionary<string, EmptyStruct>
{
{ "6379", default },
{ "8001", default }
},
});
await client.Containers.StartContainerAsync(
container.ID,
new ContainerStartParameters());
return container.ID;
}
/// <summary>
/// Stop and delete the container with the specified id.
/// </summary>
/// <param name="client">The docker client to delete the container in.</param>
/// <param name="containerId">The id of the container to delete.</param>
/// <returns>An async task.</returns>
public static async Task DeleteContainerAsync(DockerClient client, string containerId)
{
await client.Containers.StopContainerAsync(containerId, new ContainerStopParameters());
await client.Containers.RemoveContainerAsync(containerId, new ContainerRemoveParameters());
}
}
@@ -0,0 +1,67 @@
// Copyright (c) Microsoft. All rights reserved.
using Docker.DotNet;
using Npgsql;
namespace Memory.VectorStoreFixtures;
/// <summary>
/// Fixture to use for creating a Postgres container before tests and delete it after tests.
/// </summary>
public class VectorStorePostgresContainerFixture : IAsyncLifetime
{
private DockerClient? _dockerClient;
private string? _postgresContainerId;
public async Task InitializeAsync()
{
}
public async Task ManualInitializeAsync()
{
if (this._postgresContainerId == null)
{
// Connect to docker and start the docker container.
using var dockerClientConfiguration = new DockerClientConfiguration();
this._dockerClient = dockerClientConfiguration.CreateClient();
this._postgresContainerId = await VectorStoreInfra.SetupPostgresContainerAsync(this._dockerClient);
// Delay until the Postgres server is ready.
var connectionString = "Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;";
var succeeded = false;
var attemptCount = 0;
while (!succeeded && attemptCount++ < 10)
{
try
{
NpgsqlDataSourceBuilder dataSourceBuilder = new(connectionString);
dataSourceBuilder.UseVector();
using var dataSource = dataSourceBuilder.Build();
NpgsqlConnection connection = await dataSource.OpenConnectionAsync().ConfigureAwait(false);
await using (connection)
{
// Create extension vector if it doesn't exist
await using (NpgsqlCommand command = new("CREATE EXTENSION IF NOT EXISTS vector", connection))
{
await command.ExecuteNonQueryAsync();
}
}
}
catch (Exception)
{
await Task.Delay(1000);
}
}
}
}
public async Task DisposeAsync()
{
if (this._dockerClient != null && this._postgresContainerId != null)
{
// Delete docker container.
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._postgresContainerId);
}
}
}
@@ -0,0 +1,56 @@
// Copyright (c) Microsoft. All rights reserved.
using Docker.DotNet;
using Qdrant.Client;
namespace Memory.VectorStoreFixtures;
/// <summary>
/// Fixture to use for creating a Qdrant container before tests and delete it after tests.
/// </summary>
public class VectorStoreQdrantContainerFixture : IAsyncLifetime
{
private DockerClient? _dockerClient;
private string? _qdrantContainerId;
public async Task InitializeAsync()
{
}
public async Task ManualInitializeAsync()
{
if (this._qdrantContainerId == null)
{
// Connect to docker and start the docker container.
using var dockerClientConfiguration = new DockerClientConfiguration();
this._dockerClient = dockerClientConfiguration.CreateClient();
this._qdrantContainerId = await VectorStoreInfra.SetupQdrantContainerAsync(this._dockerClient);
// Delay until the Qdrant server is ready.
var qdrantClient = new QdrantClient("localhost");
var succeeded = false;
var attemptCount = 0;
while (!succeeded && attemptCount++ < 10)
{
try
{
await qdrantClient.ListCollectionsAsync();
succeeded = true;
}
catch (Exception)
{
await Task.Delay(1000);
}
}
}
}
public async Task DisposeAsync()
{
if (this._dockerClient != null && this._qdrantContainerId != null)
{
// Delete docker container.
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._qdrantContainerId);
}
}
}
@@ -0,0 +1,38 @@
// Copyright (c) Microsoft. All rights reserved.
using Docker.DotNet;
namespace Memory.VectorStoreFixtures;
/// <summary>
/// Fixture to use for creating a Redis container before tests and delete it after tests.
/// </summary>
public class VectorStoreRedisContainerFixture : IAsyncLifetime
{
private DockerClient? _dockerClient;
private string? _redisContainerId;
public async Task InitializeAsync()
{
}
public async Task ManualInitializeAsync()
{
if (this._redisContainerId == null)
{
// Connect to docker and start the docker container.
using var dockerClientConfiguration = new DockerClientConfiguration();
this._dockerClient = dockerClientConfiguration.CreateClient();
this._redisContainerId = await VectorStoreInfra.SetupRedisContainerAsync(this._dockerClient);
}
}
public async Task DisposeAsync()
{
if (this._dockerClient != null && this._redisContainerId != null)
{
// Delete docker container.
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._redisContainerId);
}
}
}
@@ -0,0 +1,34 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Memory.VectorStoreLangchainInterop;
/// <summary>
/// Data model class that matches the data model used by Langchain.
/// This data model is not decorated with vector store attributes since instead
/// a different record definition is used with each vector store implementation.
/// </summary>
/// <remarks>
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
/// </remarks>
public class LangchainDocument<TKey>
{
/// <summary>
/// The unique identifier of the record.
/// </summary>
public TKey Key { get; set; }
/// <summary>
/// The text content for which embeddings have been generated.
/// </summary>
public string Content { get; set; }
/// <summary>
/// The source of the content. E.g. where to find the original content.
/// </summary>
public string Source { get; set; }
/// <summary>
/// The embedding for the <see cref="Content"/>.
/// </summary>
public ReadOnlyMemory<float> Embedding { get; set; }
}
@@ -0,0 +1,87 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Pinecone;
using Pinecone;
namespace Memory.VectorStoreLangchainInterop;
/// <summary>
/// Contains a factory method that can be used to create a Pinecone vector store that is compatible with datasets ingested using Langchain.
/// </summary>
/// <remarks>
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
/// </remarks>
public static class PineconeFactory
{
/// <summary>
/// Record definition that matches the storage format used by Langchain for Pinecone.
/// </summary>
private static readonly VectorStoreCollectionDefinition s_definition = new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(string)),
new VectorStoreDataProperty("Content", typeof(string)) { StorageName = "text" },
new VectorStoreDataProperty("Source", typeof(string)) { StorageName = "source" },
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 1536) { StorageName = "embedding" }
]
};
/// <summary>
/// Create a new Pinecone-backed <see cref="VectorStore"/> that can be used to read data that was ingested using Langchain.
/// </summary>
/// <param name="pineconeClient">Pinecone client that can be used to manage the collections and points in a Pinecone store.</param>
/// <returns>The <see cref="VectorStore"/>.</returns>
public static VectorStore CreatePineconeLangchainInteropVectorStore(PineconeClient pineconeClient)
=> new PineconeLangchainInteropVectorStore(new PineconeVectorStore(pineconeClient), pineconeClient);
private sealed class PineconeLangchainInteropVectorStore(
VectorStore innerStore,
PineconeClient pineconeClient)
: VectorStore
{
private readonly PineconeClient _pineconeClient = pineconeClient;
public override VectorStoreCollection<TKey, TRecord> GetCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition? definition = null)
{
if (typeof(TKey) != typeof(string) || typeof(TRecord) != typeof(LangchainDocument<string>))
{
throw new NotSupportedException("This VectorStore is only usable with string keys and LangchainDocument<string> record types");
}
// Create a Pinecone collection and pass in our custom record definition that matches
// the schema used by Langchain so that the default mapper can use the storage names
// in it, to map to the storage scheme.
return (new PineconeCollection<TKey, TRecord>(
_pineconeClient,
name,
new()
{
Definition = s_definition
}) as VectorStoreCollection<TKey, TRecord>)!;
}
public override VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null)
{
// Create a Pinecone collection and pass in our custom record definition that matches
// the schema used by Langchain so that the default mapper can use the storage names
// in it, to map to the storage scheme.
return new PineconeDynamicCollection(
_pineconeClient,
name,
new()
{
Definition = s_definition
});
}
public override object? GetService(Type serviceType, object? serviceKey = null) => innerStore.GetService(serviceType, serviceKey);
public override IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default) => innerStore.ListCollectionNamesAsync(cancellationToken);
public override Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => innerStore.CollectionExistsAsync(name, cancellationToken);
public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => innerStore.EnsureCollectionDeletedAsync(name, cancellationToken);
}
}
@@ -0,0 +1,89 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Redis;
using StackExchange.Redis;
namespace Memory.VectorStoreLangchainInterop;
/// <summary>
/// Contains a factory method that can be used to create a Redis vector store that is compatible with datasets ingested using Langchain.
/// </summary>
/// <remarks>
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
/// </remarks>
public static class RedisFactory
{
/// <summary>
/// Record definition that matches the storage format used by Langchain for Redis.
/// </summary>
private static readonly VectorStoreCollectionDefinition s_definition = new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(string)),
new VectorStoreDataProperty("Content", typeof(string)) { StorageName = "text" },
new VectorStoreDataProperty("Source", typeof(string)) { StorageName = "source" },
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 1536) { StorageName = "embedding" }
]
};
/// <summary>
/// Create a new Redis-backed <see cref="VectorStore"/> that can be used to read data that was ingested using Langchain.
/// </summary>
/// <param name="database">The redis database to read/write from.</param>
/// <returns>The <see cref="VectorStore"/>.</returns>
public static VectorStore CreateRedisLangchainInteropVectorStore(IDatabase database)
=> new RedisLangchainInteropVectorStore(new RedisVectorStore(database), database);
private sealed class RedisLangchainInteropVectorStore(
VectorStore innerStore,
IDatabase database)
: VectorStore
{
private readonly IDatabase _database = database;
public override VectorStoreCollection<TKey, TRecord> GetCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition? definition = null)
{
if (typeof(TKey) != typeof(string) || typeof(TRecord) != typeof(LangchainDocument<string>))
{
throw new NotSupportedException("This VectorStore is only usable with string keys and LangchainDocument<string> record types");
}
// Create a hash set collection, since Langchain uses redis hashes for storing records.
// Also pass in our custom record definition that matches the schema used by Langchain
// so that the default mapper can use the storage names in it, to map to the storage
// scheme.
return (new RedisHashSetCollection<TKey, TRecord>(
_database,
name,
new()
{
Definition = s_definition
}) as VectorStoreCollection<TKey, TRecord>)!;
}
public override VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null)
{
// Create a hash set collection, since Langchain uses redis hashes for storing records.
// Also pass in our custom record definition that matches the schema used by Langchain
// so that the default mapper can use the storage names in it, to map to the storage
// scheme.
return new RedisHashSetDynamicCollection(
_database,
name,
new()
{
Definition = s_definition
});
}
public override object? GetService(Type serviceType, object? serviceKey = null) => innerStore.GetService(serviceType, serviceKey);
public override IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default) => innerStore.ListCollectionNamesAsync(cancellationToken);
public override Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => innerStore.CollectionExistsAsync(name, cancellationToken);
public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => innerStore.EnsureCollectionDeletedAsync(name, cancellationToken);
}
}
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text;
using System.Text.Json;
using Azure;
using Azure.Search.Documents.Indexes;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
using Microsoft.SemanticKernel.Memory;
namespace Memory;
/// <summary>
/// An example showing how use the VectorStore abstractions to consume data from an Azure AI Search data store,
/// that was created using the MemoryStore abstractions.
/// </summary>
/// <remarks>
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
/// e.g. it only supports a single fixed schema and does not allow search filtering.
/// To provide more flexibility, the Vector Store abstraction has been introduced.
///
/// To run this sample, you need an instance of Azure AI Search available and configured.
/// dotnet user-secrets set "AzureAISearch:Endpoint" "https://myazureaisearchinstance.search.windows.net"
/// dotnet user-secrets set "AzureAISearch:ApiKey" "samplesecret"
/// </remarks>
public class VectorStore_ConsumeFromMemoryStore_AzureAISearch(ITestOutputHelper output) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
private const int VectorSize = 1536;
private static readonly JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };
[Fact]
public async Task ConsumeExampleAsync()
{
// Construct a VectorStore.
var vectorStore = new AzureAISearchVectorStore(new SearchIndexClient(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey)));
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
var collection = vectorStore.GetCollection<string, VectorStoreRecord>("memorystorecollection");
await collection.EnsureCollectionExistsAsync();
// Show that the data can be read using the VectorStore abstraction.
// Note that AzureAISearchMemoryStore converts all keys to base64
// strings on upload so we need to encode the ids here before doing a get.
var record1 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("11111111-1111-1111-1111-111111111111")));
var record2 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("22222222-2222-2222-2222-222222222222")));
var record3 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("33333333-3333-3333-3333-333333333333")), new() { IncludeVectors = true });
Console.WriteLine($"Record 1: {JsonSerializer.Serialize(record1, s_consoleFormatting)}");
Console.WriteLine($"Record 2: {JsonSerializer.Serialize(record2, s_consoleFormatting)}");
Console.WriteLine($"Record 3: {JsonSerializer.Serialize(record3, s_consoleFormatting)}");
}
/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <c>AzureAISearchMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
[VectorStoreKey]
public string Id { get; set; }
[VectorStoreData]
public string Description { get; set; }
[VectorStoreData]
public string Text { get; set; }
[VectorStoreData]
public bool IsReference { get; set; }
[VectorStoreData]
public string ExternalSourceName { get; set; }
[VectorStoreData]
public string AdditionalMetadata { get; set; }
[VectorStoreVector(VectorSize)]
public ReadOnlyMemory<float> Embedding { get; set; }
}
}
@@ -0,0 +1,82 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Memory;
using Qdrant.Client;
namespace Memory;
/// <summary>
/// An example showing how use the VectorStore abstractions to consume data from a Qdrant data store,
/// that was created using the MemoryStore abstractions.
/// </summary>
/// <remarks>
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
/// e.g. it only supports a single fixed schema and does not allow search filtering.
/// To provide more flexibility, the Vector Store abstraction has been introduced.
///
/// To run this sample, you need a local instance of Docker running, since the associated fixture
/// will try and start a Qdrant container in the local docker instance to run against.
/// </remarks>
public class VectorStore_ConsumeFromMemoryStore_Qdrant(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
private const int VectorSize = 1536;
private static readonly JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };
[Fact]
public async Task ConsumeExampleAsync()
{
// Setup the supporting infra and embedding generation.
await qdrantFixture.ManualInitializeAsync();
// Construct a VectorStore.
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
var collection = vectorStore.GetCollection<Guid, VectorStoreRecord>("memorystorecollection");
await collection.EnsureCollectionExistsAsync();
// Show that the data can be read using the VectorStore abstraction.
var record1 = await collection.GetAsync(new Guid("11111111-1111-1111-1111-111111111111"));
var record2 = await collection.GetAsync(new Guid("22222222-2222-2222-2222-222222222222"));
var record3 = await collection.GetAsync(new Guid("33333333-3333-3333-3333-333333333333"), new() { IncludeVectors = true });
Console.WriteLine($"Record 1: {JsonSerializer.Serialize(record1, s_consoleFormatting)}");
Console.WriteLine($"Record 2: {JsonSerializer.Serialize(record2, s_consoleFormatting)}");
Console.WriteLine($"Record 3: {JsonSerializer.Serialize(record3, s_consoleFormatting)}");
}
/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <c>QdrantMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
[VectorStoreKey]
public Guid Key { get; set; }
[VectorStoreData(StorageName = "id")]
public string Id { get; set; }
[VectorStoreData(StorageName = "description")]
public string Description { get; set; }
[VectorStoreData(StorageName = "text")]
public string Text { get; set; }
[VectorStoreData(StorageName = "is_reference")]
public bool IsReference { get; set; }
[VectorStoreData(StorageName = "external_source_name")]
public string ExternalSourceName { get; set; }
[VectorStoreData(StorageName = "additional_metadata")]
public string AdditionalMetadata { get; set; }
[VectorStoreVector(VectorSize)]
public ReadOnlyMemory<float> Embedding { get; set; }
}
}
@@ -0,0 +1,72 @@
// Copyright (c) Microsoft. All rights reserved.
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Redis;
using Microsoft.SemanticKernel.Memory;
using StackExchange.Redis;
namespace Memory;
/// <summary>
/// An example showing how use the VectorStore abstractions to consume data from a Redis data store,
/// that was created using the MemoryStore abstractions.
/// </summary>
/// <remarks>
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
/// e.g. it only supports a single fixed schema and does not allow search filtering.
/// To provide more flexibility, the Vector Store abstraction has been introduced.
///
/// To run this sample, you need a local instance of Docker running, since the associated fixture
/// will try and start a Redis container in the local docker instance to run against.
/// </remarks>
public class VectorStore_ConsumeFromMemoryStore_Redis(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>
{
private const int VectorSize = 1536;
private const string MemoryStoreCollectionName = "memorystorecollection";
[Fact]
public async Task ConsumeExampleAsync()
{
// Setup the supporting infra and embedding generation.
await redisFixture.ManualInitializeAsync();
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction.
// Note that we use HashSet since the legacy memory store uses hashes to store memory records.
var vectorStore = new RedisVectorStore(
ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(),
new() { StorageType = RedisStorageType.HashSet });
// Connect to the same collection using the VectorStore abstraction.
var collection = vectorStore.GetCollection<string, VectorStoreRecord>(MemoryStoreCollectionName);
await collection.EnsureCollectionExistsAsync();
// Show that the data can be read using the VectorStore abstraction.
var record1 = await collection.GetAsync("11111111-1111-1111-1111-111111111111");
var record2 = await collection.GetAsync("22222222-2222-2222-2222-222222222222");
var record3 = await collection.GetAsync("33333333-3333-3333-3333-333333333333", new() { IncludeVectors = true });
Console.WriteLine($"Record 1: Key: {record1!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record1.Timestamp)} Metadata: {record1.Metadata} Embedding {record1.Embedding}");
Console.WriteLine($"Record 2: Key: {record2!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record2.Timestamp)} Metadata: {record2.Metadata} Embedding {record2.Embedding}");
Console.WriteLine($"Record 3: Key: {record3!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record3.Timestamp)} Metadata: {record3.Metadata} Embedding {record3.Embedding}");
}
/// <summary>
/// A data model with Vector Store attributes that matches the storage representation of
/// <see cref="MemoryRecord"/> objects as created by <c>RedisMemoryStore</c>.
/// </summary>
private sealed class VectorStoreRecord
{
[VectorStoreKey]
public string Key { get; set; }
[VectorStoreData(StorageName = "metadata")]
public string Metadata { get; set; }
[VectorStoreData(StorageName = "timestamp")]
public long Timestamp { get; set; }
[VectorStoreVector(VectorSize, StorageName = "embedding")]
public ReadOnlyMemory<float> Embedding { get; set; }
}
}
@@ -0,0 +1,259 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.InMemory;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Connectors.Redis;
using Qdrant.Client;
using StackExchange.Redis;
namespace Memory;
/// <summary>
/// An example showing how to ingest data into a vector store using <see cref="RedisVectorStore"/>, <see cref="QdrantVectorStore"/> or <see cref="InMemoryVectorStore"/>.
/// Since Redis and InMemory supports string keys and Qdrant supports ulong or Guid keys, this example also shows how you can have common code
/// that works with both types of keys by using a generic key generator function.
///
/// The example shows the following steps:
/// 1. Register a vector store and embedding generator with the DI container.
/// 2. Register a class (DataIngestor) with the DI container that uses the vector store and embedding generator to ingest data.
/// 3. Ingest some data into the vector store.
/// 4. Read the data back from the vector store.
///
/// For some databases in this sample (Redis &amp; Qdrant), you need a local instance of Docker running, since the associated fixtures will try and start containers in the local docker instance to run against.
/// </summary>
[Collection("Sequential")]
public class VectorStore_DataIngestion_MultiStore(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>, IClassFixture<VectorStoreQdrantContainerFixture>
{
/// <summary>
/// Example with dependency injection.
/// </summary>
/// <param name="databaseType">The type of database to run the example for.</param>
[Theory]
[InlineData("Redis")]
[InlineData("Qdrant")]
[InlineData("InMemory")]
public async Task ExampleWithDIAsync(string databaseType)
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
new AzureCliCredential(),
dimensions: 1536);
// Register the chosen vector store with the DI container and initialize docker containers via the fixtures where needed.
if (databaseType == "Redis")
{
await redisFixture.ManualInitializeAsync();
kernelBuilder.Services.AddRedisVectorStore("localhost:6379");
}
else if (databaseType == "Qdrant")
{
await qdrantFixture.ManualInitializeAsync();
kernelBuilder.Services.AddQdrantVectorStore("localhost", https: false);
}
else if (databaseType == "InMemory")
{
kernelBuilder.Services.AddInMemoryVectorStore();
}
// Register the DataIngestor with the DI container.
kernelBuilder.Services.AddTransient<DataIngestor>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a DataIngestor object using the DI container.
var dataIngestor = kernel.GetRequiredService<DataIngestor>();
// Invoke the data ingestor using an appropriate key generator function for each database type.
// Redis and InMemory supports string keys, while Qdrant supports ulong or Guid keys, so we use a different key generator for each key type.
if (databaseType is "Redis" or "InMemory")
{
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid().ToString());
}
else if (databaseType == "Qdrant")
{
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid());
}
}
/// <summary>
/// Example without dependency injection.
/// </summary>
/// <param name="databaseType">The type of database to run the example for.</param>
[Theory]
[InlineData("Redis")]
[InlineData("Qdrant")]
[InlineData("InMemory")]
public async Task ExampleWithoutDIAsync(string databaseType)
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct the chosen vector store and initialize docker containers via the fixtures where needed.
VectorStore vectorStore;
if (databaseType == "Redis")
{
await redisFixture.ManualInitializeAsync();
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
vectorStore = new RedisVectorStore(database);
}
else if (databaseType == "Qdrant")
{
await qdrantFixture.ManualInitializeAsync();
var qdrantClient = new QdrantClient("localhost", https: false);
vectorStore = new QdrantVectorStore(qdrantClient, ownsClient: true);
}
else if (databaseType == "InMemory")
{
vectorStore = new InMemoryVectorStore();
}
else
{
throw new ArgumentException("Invalid database type.");
}
// Create the DataIngestor.
var dataIngestor = new DataIngestor(vectorStore, embeddingGenerator);
// Invoke the data ingestor using an appropriate key generator function for each database type.
// Redis and InMemory supports string keys, while Qdrant supports ulong or Guid keys, so we use a different key generator for each key type.
if (databaseType is "Redis" or "InMemory")
{
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid().ToString());
}
else if (databaseType == "Qdrant")
{
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid());
}
}
private async Task UpsertDataAndReadFromVectorStoreAsync<TKey>(DataIngestor dataIngestor, Func<TKey> uniqueKeyGenerator)
where TKey : notnull
{
// Ingest some data into the vector store.
var upsertedKeys = await dataIngestor.ImportDataAsync(uniqueKeyGenerator);
// Get one of the upserted records.
var upsertedRecord = await dataIngestor.GetGlossaryAsync(upsertedKeys.First());
// Write upserted keys and one of the upserted records to the console.
Console.WriteLine($"Upserted keys: {string.Join(", ", upsertedKeys)}");
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord)}");
}
/// <summary>
/// Sample class that does ingestion of sample data into a vector store and allows retrieval of data from the vector store.
/// </summary>
/// <param name="vectorStore">The vector store to ingest data into.</param>
/// <param name="embeddingGenerator">Used to generate embeddings for the data being ingested.</param>
private sealed class DataIngestor(VectorStore vectorStore, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator)
{
/// <summary>
/// Create some glossary entries and upsert them into the vector store.
/// </summary>
/// <returns>The keys of the upserted glossary entries.</returns>
/// <typeparam name="TKey">The type of the keys in the vector store.</typeparam>
public async Task<IEnumerable<TKey>> ImportDataAsync<TKey>(Func<TKey> uniqueKeyGenerator)
where TKey : notnull
{
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>("skglossary");
await collection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateGlossaryEntries(uniqueKeyGenerator).ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection and return their keys.
await collection.UpsertAsync(glossaryEntries);
return glossaryEntries.Select(entry => entry.Key);
}
/// <summary>
/// Get a glossary entry from the vector store.
/// </summary>
/// <param name="key">The key of the glossary entry to retrieve.</param>
/// <returns>The glossary entry.</returns>
/// <typeparam name="TKey">The type of the keys in the vector store.</typeparam>
public Task<Glossary<TKey>?> GetGlossaryAsync<TKey>(TKey key)
where TKey : notnull
{
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>("skglossary");
return collection.GetAsync(key, new() { IncludeVectors = true });
}
}
/// <summary>
/// Create some sample glossary entries.
/// </summary>
/// <typeparam name="TKey">The type of the model key.</typeparam>
/// <param name="uniqueKeyGenerator">A function that can be used to generate unique keys for the model in the type that the model requires.</param>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary<TKey>> CreateGlossaryEntries<TKey>(Func<TKey> uniqueKeyGenerator)
{
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
/// <typeparam name="TKey">The type of the model key.</typeparam>
private sealed class Glossary<TKey>
{
[VectorStoreKey]
public TKey Key { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
}
@@ -0,0 +1,112 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Qdrant.Client;
namespace Memory;
/// <summary>
/// A simple example showing how to ingest data into a vector store using <see cref="QdrantVectorStore"/>.
///
/// The example shows the following steps:
/// 1. Create an embedding generator.
/// 2. Create a Qdrant Vector Store.
/// 3. Ingest some data into the vector store.
/// 4. Read the data back from the vector store.
///
/// You need a local instance of Docker running, since the associated fixture will try and start a Qdrant container in the local docker instance to run against.
/// </summary>
[Collection("Sequential")]
public class VectorStore_DataIngestion_Simple(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
[Fact]
public async Task ExampleAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initiate the docker container and construct the vector store.
await qdrantFixture.ManualInitializeAsync();
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
await collection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateGlossaryEntries().ToList();
var keys = glossaryEntries.Select(entry => entry.Key).ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection and return their keys.
await collection.UpsertAsync(glossaryEntries);
// Retrieve one of the upserted records from the collection.
var upsertedRecord = await collection.GetAsync(keys.First(), new() { IncludeVectors = true });
// Write upserted keys and one of the upserted records to the console.
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord)}");
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class Glossary
{
[VectorStoreKey]
public ulong Key { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
/// <summary>
/// Create some sample glossary entries.
/// </summary>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary> CreateGlossaryEntries()
{
yield return new Glossary
{
Key = 1,
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Glossary
{
Key = 2,
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Glossary
{
Key = 3,
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
}
@@ -0,0 +1,189 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Qdrant.Client;
namespace Memory;
/// <summary>
/// Semantic Kernel support dynamic data modeling for vector stores that can be used with any
/// schema. The schema still has to be provided in the form of a record definition, but no
/// custom .NET data model is required; a simple dictionary can be used.
///
/// The sample shows how to
/// 1. Upsert data using dynamic data modeling and retrieve it from the vector store using a custom data model.
/// 2. Upsert data using a custom data model and retrieve it from the vector store using the dynamic data modeling.
/// </summary>
public class VectorStore_DynamicDataModel_Interop(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
private static readonly JsonSerializerOptions s_indentedSerializerOptions = new() { WriteIndented = true };
private static readonly VectorStoreCollectionDefinition s_definition = new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(ulong)),
new VectorStoreDataProperty("Term", typeof(string)),
new VectorStoreDataProperty("Definition", typeof(string)),
new VectorStoreVectorProperty("DefinitionEmbedding", typeof(ReadOnlyMemory<float>), 1536)
]
};
[Fact]
public async Task UpsertWithDynamicRetrieveWithCustomAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initiate the docker container and construct the vector store.
await qdrantFixture.ManualInitializeAsync();
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
// Get and create collection if it doesn't exist using the dynamic data model and record definition that defines the schema.
var dynamicDataModelCollection = vectorStore.GetDynamicCollection("skglossary", s_definition);
await dynamicDataModelCollection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateDynamicGlossaryEntries().ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry["DefinitionEmbedding"] = (await embeddingGenerator.GenerateAsync((string)entry["Definition"]!)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection.
await dynamicDataModelCollection.UpsertAsync(glossaryEntries);
// Get the collection using the custom data model.
var customDataModelCollection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
// Retrieve one of the upserted records from the collection.
var upsertedRecord = await customDataModelCollection.GetAsync((ulong)glossaryEntries.First()["Key"]!, new() { IncludeVectors = true });
// Write one of the upserted records to the console.
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord, s_indentedSerializerOptions)}");
}
[Fact]
public async Task UpsertWithCustomRetrieveWithDynamicAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initiate the docker container and construct the vector store.
await qdrantFixture.ManualInitializeAsync();
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
// Get and create collection if it doesn't exist using the custom data model.
var customDataModelCollection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
await customDataModelCollection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateCustomGlossaryEntries().ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection and return their keys.
await customDataModelCollection.UpsertAsync(glossaryEntries);
// Get the collection using the dynamic data model.
var dynamicDataModelCollection = vectorStore.GetDynamicCollection("skglossary", s_definition);
// Retrieve one of the upserted records from the collection.
var upsertedRecord = await dynamicDataModelCollection.GetAsync(glossaryEntries.First().Key, new() { IncludeVectors = true });
// Write one of the upserted records to the console.
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord, s_indentedSerializerOptions)}");
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class Glossary
{
[VectorStoreKey]
public ulong Key { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
/// <summary>
/// Create some sample glossary entries using the custom data model.
/// </summary>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary> CreateCustomGlossaryEntries()
{
yield return new Glossary
{
Key = 1,
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data.",
};
yield return new Glossary
{
Key = 2,
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc.",
};
yield return new Glossary
{
Key = 3,
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt).",
};
}
/// <summary>
/// Create some sample glossary entries using dynamic data modeling.
/// </summary>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Dictionary<string, object?>> CreateDynamicGlossaryEntries()
{
yield return new Dictionary<string, object?>
{
["Key"] = 1ul,
["Term"] = "API",
["Definition"] = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Dictionary<string, object?>
{
["Key"] = 2ul,
["Term"] = "Connectors",
["Definition"] = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Dictionary<string, object?>
{
["Key"] = 3ul,
["Term"] = "RAG",
["Definition"] = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
}
@@ -0,0 +1,140 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure;
using Azure.AI.OpenAI;
using Azure.Identity;
using Azure.Search.Documents.Indexes;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
namespace Memory;
/// <summary>
/// A simple example showing how to ingest data into a vector store and then use hybrid search to find related records to a given string and set of keywords.
///
/// The example shows the following steps:
/// 1. Create an embedding generator.
/// 2. Create an AzureAISearch Vector Store.
/// 3. Ingest some data into the vector store.
/// 4. Do a hybrid search on the vector store with various text+keyword and filtering options.
/// </summary>
public class VectorStore_HybridSearch_Simple_AzureAISearch(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task IngestDataAndUseHybridSearch()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct the AzureAISearch VectorStore.
var searchIndexClient = new SearchIndexClient(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
var vectorStore = new AzureAISearchVectorStore(searchIndexClient);
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
await collection.EnsureCollectionExistsAsync();
var hybridSearchCollection = (IKeywordHybridSearchable<Glossary>)collection;
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateGlossaryEntries().ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection and return their keys.
await collection.UpsertAsync(glossaryEntries);
// Search the collection using a vector search.
var searchString = "What is an Application Programming Interface";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Application", "Programming", "Interface"], top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
Console.WriteLine();
// Search the collection using a vector search.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Retrieval", "Augmented", "Generation"], top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
Console.WriteLine();
// Search the collection using a vector search with pre-filtering.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Retrieval", "Augmented", "Generation"], top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Number of results: " + resultRecords.Count);
Console.WriteLine("Result 1 Score: " + resultRecords[0].Score);
Console.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
Console.WriteLine("Result 2 Score: " + resultRecords[1].Score);
Console.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class Glossary
{
[VectorStoreKey]
public string Key { get; set; }
[VectorStoreData(IsIndexed = true)]
public string Category { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData(IsFullTextIndexed = true)]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
/// <summary>
/// Create some sample glossary entries.
/// </summary>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary> CreateGlossaryEntries()
{
yield return new Glossary
{
Key = "1",
Category = "External Definitions",
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Glossary
{
Key = "2",
Category = "Core Definitions",
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Glossary
{
Key = "3",
Category = "External Definitions",
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
}
@@ -0,0 +1,76 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreLangchainInterop;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Pinecone;
using StackExchange.Redis;
namespace Memory;
/// <summary>
/// Example showing how to consume data that had previously been ingested into a database using Langchain.
/// The example also demonstrates how to get all vector stores to share the same data model, so where necessary
/// a conversion is done, specifically for ids, where the database requires GUIDs, but we want to use strings
/// containing GUIDs in the common data model.
/// </summary>
/// <remarks>
/// To run these samples, you need to first create collections instances using Langhain.
/// This sample assumes that you used the pets sample data set from this article:
/// https://python.langchain.com/docs/tutorials/retrievers/#documents
/// And the from_documents method to create the collection as shown here:
/// https://python.langchain.com/docs/tutorials/retrievers/#vector-stores
/// </remarks>
public class VectorStore_Langchain_Interop(ITestOutputHelper output) : BaseTest(output)
{
/// <summary>
/// Shows how to read data from a Pinecone collection that was created and ingested using Langchain.
/// </summary>
[Fact]
public async Task ReadDataFromLangchainPineconeAsync()
{
var pineconeClient = new PineconeClient(TestConfiguration.Pinecone.ApiKey);
var vectorStore = PineconeFactory.CreatePineconeLangchainInteropVectorStore(pineconeClient);
await this.ReadDataFromCollectionAsync(vectorStore, "pets");
}
/// <summary>
/// Shows how to read data from a Redis collection that was created and ingested using Langchain.
/// </summary>
[Fact]
public async Task ReadDataFromLangchainRedisAsync()
{
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
var vectorStore = RedisFactory.CreateRedisLangchainInteropVectorStore(database);
await this.ReadDataFromCollectionAsync(vectorStore, "pets");
}
/// <summary>
/// Method to do a vector search on a collection in the provided vector store.
/// </summary>
/// <param name="vectorStore">The vector store to search.</param>
/// <param name="collectionName">The name of the collection.</param>
/// <returns>An async task.</returns>
private async Task ReadDataFromCollectionAsync(VectorStore vectorStore, string collectionName)
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator();
// Get the collection.
var collection = vectorStore.GetCollection<string, LangchainDocument<string>>(collectionName);
// Search the data set.
var searchString = "I'm looking for an animal that is loyal and will make a great companion";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
this.Output.WriteLine("Search string: " + searchString);
this.Output.WriteLine("Source: " + resultRecords.First().Record.Source);
this.Output.WriteLine("Text: " + resultRecords.First().Record.Content);
this.Output.WriteLine();
}
}
@@ -0,0 +1,90 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure;
using Azure.AI.OpenAI;
using Azure.Identity;
using Azure.Search.Documents.Indexes;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
namespace Memory;
/// <summary>
/// An example showing how to use common code, that can work with any vector database, with an Azure AI Search instance.
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
/// The common code ingests data into the vector store and then searches over that data.
/// This example is part of a set of examples each showing a different vector database.
///
/// For other databases, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
///
/// To run this sample, you need an already existing Azure AI Search instance.
/// To set your secrets use:
/// <para> dotnet user-secrets set "AzureAISearch:Endpoint" "https://... .search.windows.net"</para>
/// <para> dotnet user-secrets set "AzureAISearch:ApiKey" "{Key from your Search service resource}"</para>
/// </summary>
public class VectorStore_VectorSearch_MultiStore_AzureAISearch(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task ExampleWithDIAsync()
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
credential: new AzureCliCredential(),
dimensions: 1536);
// Register the Azure AI Search VectorStore.
kernelBuilder.Services.AddAzureAISearchVectorStore(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a common processor object using the DI container.
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Azure AI Search supports string, but others may not support strings.
await processor.IngestDataAndSearchAsync("skglossary-with-di", () => Guid.NewGuid().ToString());
}
[Fact]
public async Task ExampleWithoutDIAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct the Azure AI Search VectorStore.
var searchIndexClient = new SearchIndexClient(
new Uri(TestConfiguration.AzureAISearch.Endpoint),
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
var vectorStore = new AzureAISearchVectorStore(searchIndexClient);
// Create the common processor that works for any vector store.
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Azure AI Search supports string, but others may not support strings.
await processor.IngestDataAndSearchAsync("skglossary-without-di", () => Guid.NewGuid().ToString());
}
}
@@ -0,0 +1,141 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
namespace Memory;
/// <summary>
/// This class is part of an example that shows how to ingest data into a vector store and then use vector search to find related records to a given string.
/// The example shows how to write code that can be used with multiple database types.
/// This class contains the common code.
///
/// For the entry point of the example for each database, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
/// </summary>
/// <param name="vectorStore">The vector store to ingest data into.</param>
/// <param name="embeddingGenerator">The service to use for generating embeddings.</param>
/// <param name="output">A helper to write output to the xUnit test output stream.</param>
public class VectorStore_VectorSearch_MultiStore_Common(VectorStore vectorStore, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, ITestOutputHelper output)
{
/// <summary>
/// Ingest data into a collection with the given name, and search over that data.
/// </summary>
/// <typeparam name="TKey">The type of key to use for database records.</typeparam>
/// <param name="collectionName">The name of the collection to ingest the data into.</param>
/// <param name="uniqueKeyGenerator">A function to generate unique keys for each record to upsert.</param>
/// <returns>An async task.</returns>
public async Task IngestDataAndSearchAsync<TKey>(string collectionName, Func<TKey> uniqueKeyGenerator)
where TKey : notnull
{
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>(collectionName);
await collection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateGlossaryEntries(uniqueKeyGenerator).ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection.
await collection.UpsertAsync(glossaryEntries);
await Task.Delay(5000); // Add a wait to ensure that indexing completes before we continue.
// Search the collection using a vector search.
var searchString = "What is an Application Programming Interface";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
output.WriteLine("Search string: " + searchString);
output.WriteLine("Result: " + resultRecords.First().Record.Definition);
output.WriteLine();
// Search the collection using a vector search.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
output.WriteLine("Search string: " + searchString);
output.WriteLine("Result: " + resultRecords.First().Record.Definition);
output.WriteLine();
// Search the collection using a vector search with pre-filtering.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await collection.SearchAsync(searchVector, top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
output.WriteLine("Search string: " + searchString);
output.WriteLine("Number of results: " + resultRecords.Count);
output.WriteLine("Result 1 Score: " + resultRecords[0].Score);
output.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
output.WriteLine("Result 2 Score: " + resultRecords[1].Score);
output.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
}
/// <summary>
/// Create some sample glossary entries.
/// </summary>
/// <typeparam name="TKey">The type of the model key.</typeparam>
/// <param name="uniqueKeyGenerator">A function that can be used to generate unique keys for the model in the type that the model requires.</param>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary<TKey>> CreateGlossaryEntries<TKey>(Func<TKey> uniqueKeyGenerator)
{
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Category = "External Definitions",
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Category = "Core Definitions",
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Glossary<TKey>
{
Key = uniqueKeyGenerator(),
Category = "External Definitions",
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
/// <typeparam name="TKey">The type of the model key.</typeparam>
private sealed class Glossary<TKey>
{
[VectorStoreKey]
public TKey Key { get; set; }
[VectorStoreData(IsIndexed = true)]
public string Category { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
}
@@ -0,0 +1,82 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.InMemory;
namespace Memory;
/// <summary>
/// An example showing how to use common code, that can work with any vector database, with the InMemory vector store.
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
/// The common code ingests data into the vector store and then searches over that data.
/// This example is part of a set of examples each showing a different vector database.
///
/// For other databases, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
/// </summary>
public class VectorStore_VectorSearch_MultiStore_InMemory(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task ExampleWithDIAsync()
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
credential: new AzureCliCredential(),
dimensions: 1536);
// Register the InMemory VectorStore.
kernelBuilder.Services.AddInMemoryVectorStore();
// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a common processor object using the DI container.
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. InMemory supports any comparable type, but others may only support string or Guid or ulong, etc.
// For this example we'll use int.
var uniqueId = 0;
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => uniqueId++);
}
[Fact]
public async Task ExampleWithoutDIAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct the InMemory VectorStore.
var vectorStore = new InMemoryVectorStore();
// Create the common processor that works for any vector store.
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. InMemory supports any comparable type, but others may only support string or Guid or ulong, etc.
// For this example we'll use int.
var uniqueId = 0;
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => uniqueId++);
}
}
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.PgVector;
namespace Memory;
/// <summary>
/// An example showing how to use common code, that can work with any vector database, with a Postgres database.
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
/// The common code ingests data into the vector store and then searches over that data.
/// This example is part of a set of examples each showing a different vector database.
///
/// For other databases, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
///
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Postgres container in the local docker instance.
/// </summary>
public class VectorStore_VectorSearch_MultiStore_Postgres(ITestOutputHelper output, VectorStorePostgresContainerFixture PostgresFixture) : BaseTest(output), IClassFixture<VectorStorePostgresContainerFixture>
{
[Fact]
public async Task ExampleWithDIAsync()
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
credential: new AzureCliCredential(),
dimensions: 1536);
// Initialize the Postgres docker container via the fixtures and register the Postgres VectorStore.
await PostgresFixture.ManualInitializeAsync();
kernelBuilder.Services.AddPostgresVectorStore("Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;");
// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a common processor object using the DI container.
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Postgres supports Guid and ulong keys, but others may support strings only.
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => Guid.NewGuid());
}
[Fact]
public async Task ExampleWithoutDIAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initialize the Postgres docker container via the fixtures and construct the Postgres VectorStore.
await PostgresFixture.ManualInitializeAsync();
using PostgresVectorStore vectorStore = new("Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;");
// Create the common processor that works for any vector store.
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Postgres supports Guid and ulong keys, but others may support strings only.
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => Guid.NewGuid());
}
}
@@ -0,0 +1,85 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Qdrant.Client;
namespace Memory;
/// <summary>
/// An example showing how to use common code, that can work with any vector database, with a Qdrant database.
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
/// The common code ingests data into the vector store and then searches over that data.
/// This example is part of a set of examples each showing a different vector database.
///
/// For other databases, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
///
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Qdrant container in the local docker instance.
/// </summary>
public class VectorStore_VectorSearch_MultiStore_Qdrant(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
{
[Fact]
public async Task ExampleWithDIAsync()
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
credential: new AzureCliCredential(),
dimensions: 1536);
// Initialize the Qdrant docker container via the fixtures and register the Qdrant VectorStore.
await qdrantFixture.ManualInitializeAsync();
kernelBuilder.Services.AddQdrantVectorStore("localhost", https: false);
// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a common processor object using the DI container.
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Qdrant supports Guid and ulong keys, but others may support strings only.
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => Guid.NewGuid());
}
[Fact]
public async Task ExampleWithoutDIAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initialize the Qdrant docker container via the fixtures and construct the Qdrant VectorStore.
await qdrantFixture.ManualInitializeAsync();
var qdrantClient = new QdrantClient("localhost");
var vectorStore = new QdrantVectorStore(qdrantClient, ownsClient: true);
// Create the common processor that works for any vector store.
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Qdrant supports Guid and ulong keys, but others may support strings only.
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => Guid.NewGuid());
}
}
@@ -0,0 +1,96 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Memory.VectorStoreFixtures;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.Redis;
using StackExchange.Redis;
namespace Memory;
/// <summary>
/// An example showing how to use common code, that can work with any vector database, with a Redis database.
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
/// The common code ingests data into the vector store and then searches over that data.
/// This example is part of a set of examples each showing a different vector database.
///
/// For other databases, see the following classes:
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
///
/// Redis supports two record storage types: Json and HashSet.
/// Note the use of the <see cref="RedisStorageType"/> enum to specify the preferred storage type.
///
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Redis container in the local docker instance.
/// </summary>
public class VectorStore_VectorSearch_MultiStore_Redis(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>
{
[Theory]
[InlineData(RedisStorageType.Json)]
[InlineData(RedisStorageType.HashSet)]
public async Task ExampleWithDIAsync(RedisStorageType redisStorageType)
{
// Use the kernel for DI purposes.
var kernelBuilder = Kernel
.CreateBuilder();
// Register an embedding generation service with the DI container.
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
credential: new AzureCliCredential(),
dimensions: 1536);
// Initialize the Redis docker container via the fixtures and register the Redis VectorStore with the preferred storage type.
await redisFixture.ManualInitializeAsync();
kernelBuilder.Services.AddRedisVectorStore("localhost:6379", new() { StorageType = redisStorageType });
// Register the test output helper common processor with the DI container.
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
// Build the kernel.
var kernel = kernelBuilder.Build();
// Build a common processor object using the DI container.
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Redis supports string keys, but others may not support string.
// Also note that we are appending the collection name with the storage type so that we have two separate collections,
// since a redis index for JSON records cannot be used to index hashset documents, and vice versa.
await processor.IngestDataAndSearchAsync("skglossaryWithDI" + redisStorageType, () => Guid.NewGuid().ToString());
}
[Theory]
[InlineData(RedisStorageType.Json)]
[InlineData(RedisStorageType.HashSet)]
public async Task ExampleWithoutDIAsync(RedisStorageType redisStorageType)
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Initialize the Redis docker container via the fixtures and construct the Redis VectorStore with the preferred storage type.
await redisFixture.ManualInitializeAsync();
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
var vectorStore = new RedisVectorStore(database, new() { StorageType = redisStorageType });
// Create the common processor that works for any vector store.
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
// Run the process and pass a key generator function to it, to generate unique record keys.
// The key generator function is required, since different vector stores may require different key types.
// E.g. Redis supports string keys, but others may not support string.
// Also note that we are appending the collection name with the storage type so that we have two separate collections,
// since a redis index for JSON records cannot be used to index hashset documents, and vice versa.
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI" + redisStorageType, () => Guid.NewGuid().ToString());
}
}
@@ -0,0 +1,142 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.InMemory;
namespace Memory;
/// <summary>
/// An example showing how to do vector search where there may be multiple vectors
/// stored in each record and you want to specify which vector to search on.
///
/// The example shows the following steps:
/// 1. Create an InMemory Vector Store.
/// 2. Generate and add some test data entries.
/// 3. Search for records based on a specified vector.
/// </summary>
public class VectorStore_VectorSearch_MultiVector(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task VectorSearchWithMultiVectorRecordAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct an InMemory vector store.
var vectorStore = new InMemoryVectorStore();
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<int, Product>("skproducts");
await collection.EnsureCollectionExistsAsync();
// Create product records and generate embeddings for them.
var productRecords = CreateProductRecords().ToList();
var tasks = productRecords.Select(entry => Task.Run(async () =>
{
var descriptionEmbeddingTask = embeddingGenerator.GenerateAsync(entry.Description);
var featureListEmbeddingTask = embeddingGenerator.GenerateAsync(string.Join("\n", entry.FeatureList));
entry.DescriptionEmbedding = (await descriptionEmbeddingTask).Vector;
entry.FeatureListEmbedding = (await featureListEmbeddingTask).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the product records into the collection.
await collection.UpsertAsync(productRecords);
// Search the store using the description embedding.
var searchString = "I am looking for a reasonably priced coffee maker";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await collection.SearchAsync(
searchVector, top: 1, new()
{
VectorProperty = r => r.DescriptionEmbedding
}).ToListAsync();
WriteLine("Search string: " + searchString);
WriteLine("Result: " + resultRecords.First().Record.Description);
WriteLine("Score: " + resultRecords.First().Score);
WriteLine();
// Search the store using the feature list embedding.
searchString = "I am looking for a handheld vacuum cleaner that will remove pet hair";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await collection.SearchAsync(
searchVector,
top: 1,
new()
{
VectorProperty = r => r.FeatureListEmbedding
}).ToListAsync();
WriteLine("Search string: " + searchString);
WriteLine("Result: " + resultRecords.First().Record.Description);
WriteLine("Score: " + resultRecords.First().Score);
WriteLine();
}
/// <summary>
/// Create some sample product records.
/// </summary>
/// <returns>A list of sample product records.</returns>
private static IEnumerable<Product> CreateProductRecords()
{
yield return new Product
{
Key = 1,
Description = "Premium coffee maker that allows you to make up to 20 types of drinks with one machine.",
FeatureList = ["Milk Frother", "Easy to use", "One button operation", "Stylish design"]
};
yield return new Product
{
Key = 2,
Description = "Value coffee maker that gives you what you need at a good price.",
FeatureList = ["Simple design", "Easy to clean"]
};
yield return new Product
{
Key = 3,
Description = "Efficient vacuum cleaner",
FeatureList = ["1000W power", "Hard floor tool", "Bagless", "Corded"]
};
yield return new Product
{
Key = 4,
Description = "High performance handheld vacuum cleaner",
FeatureList = ["Pet hair tool", "2000W power", "Hard floor tool", "Bagless", "Cordless"]
};
}
/// <summary>
/// Sample model class that can store product information with a description and a feature list with embeddings for both.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class Product
{
[VectorStoreKey]
public int Key { get; set; }
[VectorStoreData]
public string Description { get; set; }
[VectorStoreData]
public List<string> FeatureList { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DescriptionEmbedding { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> FeatureListEmbedding { get; set; }
}
}
@@ -0,0 +1,90 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.InMemory;
namespace Memory;
/// <summary>
/// An example showing how to do paging when there are many records in the database and you want to page through these page by page.
///
/// The example shows the following steps:
/// 1. Create an InMemory Vector Store.
/// 2. Generate and add some test data entries.
/// 3. Read the data back using vector search by paging through the results page by page.
/// </summary>
public class VectorStore_VectorSearch_Paging(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task VectorSearchWithPagingAsync()
{
// Construct an InMemory vector store.
var vectorStore = new InMemoryVectorStore();
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<int, TextSnippet>("skglossary");
await collection.EnsureCollectionExistsAsync();
// Create some test data entries.
// We are not generating real embeddings here, just some random numbers
// to keep the example simple.
for (int i = 0; i < 1000; i++)
{
var text = $"This is a test text snippet {i}";
var embedding = new ReadOnlyMemory<float>([i, i + 1, i + 2, i + 3]);
var textSnippet = new TextSnippet { Key = i, Text = text, TextEmbedding = embedding };
await collection.UpsertAsync(textSnippet);
}
// Create a vector to search with.
// We are not generating a real embedding here, just some random numbers
// to keep the example simple.
var searchVector = new ReadOnlyMemory<float>([0, 1, 2, 3]);
// Loop until there are no more results.
var page = 0;
var moreResults = true;
while (moreResults)
{
// Get the next page of results by asking for 10 results, and using 'Skip' to skip the results from the previous pages.
var currentPageResults = collection.SearchAsync(
searchVector,
top: 10,
new()
{
Skip = page * 10
});
// Print the results.
var pageCount = 0;
await foreach (var result in currentPageResults)
{
Console.WriteLine($"Key: {result.Record.Key}, Text: {result.Record.Text}");
pageCount++;
}
// Stop when we got back less than the requested number of results.
moreResults = pageCount == 10;
page++;
}
}
/// <summary>
/// Sample model class that can store some text and its embedding.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class TextSnippet
{
[VectorStoreKey]
public int Key { get; set; }
[VectorStoreData]
public string Text { get; set; }
[VectorStoreVector(4)]
public ReadOnlyMemory<float> TextEmbedding { get; set; }
}
}
@@ -0,0 +1,134 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.InMemory;
namespace Memory;
/// <summary>
/// A simple example showing how to ingest data into a vector store and then use vector search to find related records to a given string.
///
/// The example shows the following steps:
/// 1. Create an embedding generator.
/// 2. Create an InMemory Vector Store.
/// 3. Ingest some data into the vector store.
/// 4. Search the vector store with various text and filtering options.
/// </summary>
public class VectorStore_VectorSearch_Simple(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task ExampleAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
.AsIEmbeddingGenerator(1536);
// Construct an InMemory vector store.
var vectorStore = new InMemoryVectorStore();
// Get and create collection if it doesn't exist.
var collection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
await collection.EnsureCollectionExistsAsync();
// Create glossary entries and generate embeddings for them.
var glossaryEntries = CreateGlossaryEntries().ToList();
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
{
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
}));
await Task.WhenAll(tasks);
// Upsert the glossary entries into the collection and return their keys.
await collection.UpsertAsync(glossaryEntries);
// Search the collection using a vector search.
var searchString = "What is an Application Programming Interface";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
Console.WriteLine();
// Search the collection using a vector search.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
Console.WriteLine();
// Search the collection using a vector search with pre-filtering.
searchString = "What is Retrieval Augmented Generation";
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
resultRecords = await collection.SearchAsync(searchVector, top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Number of results: " + resultRecords.Count);
Console.WriteLine("Result 1 Score: " + resultRecords[0].Score);
Console.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
Console.WriteLine("Result 2 Score: " + resultRecords[1].Score);
Console.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
}
/// <summary>
/// Sample model class that represents a glossary entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class Glossary
{
[VectorStoreKey]
public ulong Key { get; set; }
[VectorStoreData(IsIndexed = true)]
public string Category { get; set; }
[VectorStoreData]
public string Term { get; set; }
[VectorStoreData]
public string Definition { get; set; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
}
/// <summary>
/// Create some sample glossary entries.
/// </summary>
/// <returns>A list of sample glossary entries.</returns>
private static IEnumerable<Glossary> CreateGlossaryEntries()
{
yield return new Glossary
{
Key = 1,
Category = "External Definitions",
Term = "API",
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
};
yield return new Glossary
{
Key = 2,
Category = "Core Definitions",
Term = "Connectors",
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
};
yield return new Glossary
{
Key = 3,
Category = "External Definitions",
Term = "RAG",
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a users question (prompt)."
};
}
}
@@ -0,0 +1,151 @@
// Copyright (c) Microsoft. All rights reserved.
using System.ClientModel;
using System.ClientModel.Primitives;
using System.Text.Json;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using Microsoft.SemanticKernel.Connectors.InMemory;
using Microsoft.SemanticKernel.Data;
using Resources;
namespace Memory;
/// <summary>
/// Sample showing how to create an <see cref="InMemoryVectorStore"/> collection from a list of strings
/// and then save it to disk so that it can be reloaded later.
/// </summary>
public class InMemoryVectorStore_LoadData(ITestOutputHelper output) : BaseTest(output)
{
[Fact]
public async Task LoadStringListAndSearchAsync()
{
// Create a logging handler to output HTTP requests and responses
var handler = new LoggingHandler(new HttpClientHandler(), this.Output);
var httpClient = new HttpClient(handler);
// Create an embedding generation service.
var embeddingGenerator = new OpenAI.OpenAIClient(
new ApiKeyCredential(TestConfiguration.OpenAI.ApiKey),
new OpenAI.OpenAIClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) })
.GetEmbeddingClient(TestConfiguration.OpenAI.EmbeddingModelId)
.AsIEmbeddingGenerator(1536);
// Construct an InMemory vector store.
var vectorStore = new InMemoryVectorStore();
var collectionName = "records";
// Path to the file where the record collection will be saved to and loaded from.
string filePath = Path.Combine(Path.GetTempPath(), "semantic-kernel-info.json");
if (!File.Exists(filePath))
{
// Read a list of text strings from a file, to load into a new record collection.
var skInfo = EmbeddedResource.Read("semantic-kernel-info.txt");
var lines = skInfo!.Split('\n');
// Delegate which will create a record.
static DataModel CreateRecord(string text, ReadOnlyMemory<float> embedding)
{
return new()
{
Key = Guid.NewGuid(),
Text = text,
Embedding = embedding
};
}
// Create a record collection from a list of strings using the provided delegate.
var collection = await vectorStore.CreateCollectionFromListAsync<Guid, DataModel>(
collectionName, lines, embeddingGenerator, CreateRecord);
// Save the record collection to a file stream.
using (FileStream fileStream = new(filePath, FileMode.OpenOrCreate))
{
await vectorStore.SerializeCollectionAsJsonAsync<Guid, DataModel>(collectionName, fileStream);
}
}
// Load the record collection from the file stream and perform a search.
using (FileStream fileStream = new(filePath, FileMode.Open))
{
var vectorSearch = await vectorStore.DeserializeCollectionFromJsonAsync<Guid, DataModel>(fileStream);
// Search the collection using a vector search.
var searchString = "What is the Semantic Kernel?";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await vectorSearch!.SearchAsync(searchVector, top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Text);
Console.WriteLine();
}
}
[Fact]
public async Task LoadTextSearchResultsAndSearchAsync()
{
// Create an embedding generation service.
var embeddingGenerator = new OpenAI.OpenAIClient(TestConfiguration.OpenAI.ApiKey)
.GetEmbeddingClient(TestConfiguration.OpenAI.EmbeddingModelId)
.AsIEmbeddingGenerator(1536);
// Construct an InMemory vector store.
var vectorStore = new InMemoryVectorStore();
var collectionName = "records";
// Read a list of text strings from a file, to load into a new record collection.
var searchResultsJson = EmbeddedResource.Read("what-is-semantic-kernel.json");
var searchResults = JsonSerializer.Deserialize<List<TextSearchResult>>(searchResultsJson!);
// Delegate which will create a record.
static DataModel CreateRecord(TextSearchResult searchResult, ReadOnlyMemory<float> embedding)
{
return new()
{
Key = Guid.NewGuid(),
Title = searchResult.Name,
Text = searchResult.Value ?? string.Empty,
Link = searchResult.Link,
Embedding = embedding
};
}
// Create a record collection from a list of strings using the provided delegate.
var vectorSearch = await vectorStore.CreateCollectionFromTextSearchResultsAsync<Guid, DataModel>(
collectionName, searchResults!, embeddingGenerator, CreateRecord);
// Search the collection using a vector search.
var searchString = "What is the Semantic Kernel?";
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
var resultRecords = await vectorSearch!.SearchAsync(searchVector, top: 1).ToListAsync();
Console.WriteLine("Search string: " + searchString);
Console.WriteLine("Result: " + resultRecords.First().Record.Text);
Console.WriteLine();
}
/// <summary>
/// Sample model class that represents a record entry.
/// </summary>
/// <remarks>
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
/// </remarks>
private sealed class DataModel
{
[VectorStoreKey]
public Guid Key { get; init; }
[VectorStoreData]
public string? Title { get; init; }
[VectorStoreData]
public string Text { get; init; }
[VectorStoreData]
public string? Link { get; init; }
[VectorStoreVector(1536)]
public ReadOnlyMemory<float> Embedding { get; init; }
}
}