chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.SemanticKernel;
|
||||
using xRetry;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with AWS Bedrock API for embedding generation,
|
||||
// including the ability to specify custom dimensions.
|
||||
public class AWSBedrock_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
/// <summary>
|
||||
/// This test demonstrates how to use the AWS Bedrock API embedding generation.
|
||||
/// </summary>
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task GenerateEmbeddings()
|
||||
{
|
||||
IKernelBuilder kernelBuilder = Kernel.CreateBuilder()
|
||||
.AddBedrockEmbeddingGenerator(modelId: TestConfiguration.Bedrock.EmbeddingModelId! ?? "amazon.titan-embed-text-v1");
|
||||
|
||||
Kernel kernel = kernelBuilder.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings with the default dimensions for the model
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(
|
||||
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
|
||||
|
||||
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default for current model) for the provided text");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Google.Apis.Auth.OAuth2;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.SemanticKernel;
|
||||
using xRetry;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with Google AI and Google's Vertex AI for embedding generation,
|
||||
// including the ability to specify custom dimensions.
|
||||
public class Google_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
/// <summary>
|
||||
/// This test demonstrates how to use the Google Vertex AI embedding generation service with default dimensions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Currently custom dimensions are not supported for Vertex AI.
|
||||
/// </remarks>
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task GenerateEmbeddingWithDefaultDimensionsUsingVertexAI()
|
||||
{
|
||||
string? bearerToken = null;
|
||||
|
||||
Assert.NotNull(TestConfiguration.VertexAI.EmbeddingModelId);
|
||||
Assert.NotNull(TestConfiguration.VertexAI.ClientId);
|
||||
Assert.NotNull(TestConfiguration.VertexAI.ClientSecret);
|
||||
Assert.NotNull(TestConfiguration.VertexAI.Location);
|
||||
Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
|
||||
|
||||
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
|
||||
kernelBuilder.AddVertexAIEmbeddingGenerator(
|
||||
modelId: TestConfiguration.VertexAI.EmbeddingModelId!,
|
||||
bearerTokenProvider: GetBearerToken,
|
||||
location: TestConfiguration.VertexAI.Location,
|
||||
projectId: TestConfiguration.VertexAI.ProjectId);
|
||||
Kernel kernel = kernelBuilder.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings with the default dimensions for the model
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(
|
||||
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
|
||||
|
||||
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default) for the provided text");
|
||||
|
||||
// Uses Google.Apis.Auth.OAuth2 to get the bearer token
|
||||
async ValueTask<string> GetBearerToken()
|
||||
{
|
||||
if (!string.IsNullOrEmpty(bearerToken))
|
||||
{
|
||||
return bearerToken;
|
||||
}
|
||||
|
||||
var credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
|
||||
new ClientSecrets
|
||||
{
|
||||
ClientId = TestConfiguration.VertexAI.ClientId,
|
||||
ClientSecret = TestConfiguration.VertexAI.ClientSecret
|
||||
},
|
||||
["https://www.googleapis.com/auth/cloud-platform"],
|
||||
"user",
|
||||
CancellationToken.None);
|
||||
|
||||
var userCredential = await credential.WaitAsync(CancellationToken.None);
|
||||
bearerToken = userCredential.Token.AccessToken;
|
||||
|
||||
return bearerToken;
|
||||
}
|
||||
}
|
||||
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task GenerateEmbeddingWithDefaultDimensionsUsingGoogleAI()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
|
||||
Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
|
||||
|
||||
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
|
||||
kernelBuilder.AddGoogleAIEmbeddingGenerator(
|
||||
modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
|
||||
apiKey: TestConfiguration.GoogleAI.ApiKey);
|
||||
Kernel kernel = kernelBuilder.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings with the default dimensions for the model
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(
|
||||
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
|
||||
|
||||
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (default) for the provided text");
|
||||
}
|
||||
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task GenerateEmbeddingWithCustomDimensionsUsingGoogleAI()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
|
||||
Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
|
||||
|
||||
// Specify custom dimensions for the embeddings
|
||||
const int CustomDimensions = 512;
|
||||
|
||||
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
|
||||
kernelBuilder.AddGoogleAIEmbeddingGenerator(
|
||||
modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
|
||||
apiKey: TestConfiguration.GoogleAI.ApiKey,
|
||||
dimensions: CustomDimensions);
|
||||
Kernel kernel = kernelBuilder.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings with the specified custom dimensions
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(
|
||||
["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
|
||||
|
||||
Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Vector.Length}' dimensions (custom: '{CustomDimensions}') for the provided text");
|
||||
|
||||
// Verify that we received embeddings with our requested dimensions
|
||||
Assert.Equal(CustomDimensions, embeddings[0].Vector.Length);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.SemanticKernel;
|
||||
using xRetry;
|
||||
|
||||
#pragma warning disable format // Format item can be simplified
|
||||
#pragma warning disable CA1861 // Avoid constant arrays as arguments
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with HuggingFace API.
|
||||
public class HuggingFace_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task RunInferenceApiEmbeddingAsync()
|
||||
{
|
||||
Console.WriteLine("\n======= Hugging Face Inference API - Embedding Example ========\n");
|
||||
|
||||
Kernel kernel = Kernel.CreateBuilder()
|
||||
.AddHuggingFaceEmbeddingGenerator(
|
||||
model: TestConfiguration.HuggingFace.EmbeddingModelId,
|
||||
apiKey: TestConfiguration.HuggingFace.ApiKey)
|
||||
.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings for each chunk.
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.HuggingFace;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
using Microsoft.SemanticKernel.Embeddings;
|
||||
|
||||
#pragma warning disable CS8602 // Dereference of a possibly null reference.
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// This example shows how to use custom <see cref="HttpClientHandler"/> to override Hugging Face HTTP response.
|
||||
/// Generally, an embedding model will return results as a 1 * n matrix for input type [string]. However, the model can have different matrix dimensionality.
|
||||
/// For example, the <a href="https://huggingface.co/cointegrated/LaBSE-en-ru">cointegrated/LaBSE-en-ru</a> model returns results as a 1 * 1 * 4 * 768 matrix, which is different from Hugging Face embedding generation service implementation.
|
||||
/// To address this, a custom <see cref="HttpClientHandler"/> can be used to modify the response before sending it back.
|
||||
/// </summary>
|
||||
[Obsolete("The IMemoryStore abstraction is being obsoleted")]
|
||||
public class HuggingFace_TextEmbeddingCustomHttpHandler(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
public async Task RunInferenceApiEmbeddingCustomHttpHandlerAsync()
|
||||
{
|
||||
Console.WriteLine("\n======= Hugging Face Inference API - Embedding Example ========\n");
|
||||
|
||||
var hf = new HuggingFaceTextEmbeddingGenerationService(
|
||||
"cointegrated/LaBSE-en-ru",
|
||||
apiKey: TestConfiguration.HuggingFace.ApiKey,
|
||||
httpClient: new HttpClient(new CustomHttpClientHandler()
|
||||
{
|
||||
CheckCertificateRevocationList = true
|
||||
})
|
||||
);
|
||||
|
||||
var inMemoryCollection = new InMemoryCollection<string, Record>(
|
||||
name: "Test",
|
||||
new() { EmbeddingGenerator = hf.AsEmbeddingGenerator() });
|
||||
|
||||
await inMemoryCollection.UpsertAsync(new Record
|
||||
{
|
||||
Id = "1",
|
||||
Text = "THIS IS A SAMPLE",
|
||||
Embedding = "An embedding will be generated from this text"
|
||||
});
|
||||
}
|
||||
|
||||
public class Record
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public string Id { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Text { get; set; }
|
||||
|
||||
[VectorStoreVector(Dimensions: 768)]
|
||||
public string Embedding { get; set; }
|
||||
}
|
||||
|
||||
private sealed class CustomHttpClientHandler : HttpClientHandler
|
||||
{
|
||||
private readonly JsonSerializerOptions _jsonOptions = new();
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
// Log the request URI
|
||||
//Console.WriteLine($"Request: {request.Method} {request.RequestUri}");
|
||||
|
||||
// Send the request and get the response
|
||||
HttpResponseMessage response = await base.SendAsync(request, cancellationToken);
|
||||
|
||||
// Log the response status code
|
||||
//Console.WriteLine($"Response: {(int)response.StatusCode} {response.ReasonPhrase}");
|
||||
|
||||
// You can manipulate the response here
|
||||
// For example, add a custom header
|
||||
// response.Headers.Add("X-Custom-Header", "CustomValue");
|
||||
|
||||
// For example, modify the response content
|
||||
Stream originalContent = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
List<List<List<ReadOnlyMemory<float>>>> modifiedContent = (await JsonSerializer.DeserializeAsync<List<List<List<ReadOnlyMemory<float>>>>>(originalContent, _jsonOptions, cancellationToken).ConfigureAwait(false))!;
|
||||
|
||||
Stream modifiedStream = new MemoryStream();
|
||||
await JsonSerializer.SerializeAsync(modifiedStream, modifiedContent[0][0].ToList(), _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
response.Content = new StreamContent(modifiedStream);
|
||||
|
||||
// Return the modified response
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.SemanticKernel;
|
||||
using xRetry;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with Ollama API.
|
||||
public class Ollama_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task RunEmbeddingAsync()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.Ollama.EmbeddingModelId);
|
||||
|
||||
Console.WriteLine("\n======= Ollama - Embedding Example ========\n");
|
||||
|
||||
Kernel kernel = Kernel.CreateBuilder()
|
||||
.AddOllamaEmbeddingGenerator(
|
||||
endpoint: new Uri(TestConfiguration.Ollama.Endpoint),
|
||||
modelId: TestConfiguration.Ollama.EmbeddingModelId)
|
||||
.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings for each chunk.
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with Onnx GenAI API.
|
||||
public class Onnx_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
/// <summary>
|
||||
/// Example using the service directly to get embeddings
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Configuration example:
|
||||
/// <list type="table">
|
||||
/// <item>
|
||||
/// <term>EmbeddingModelPath:</term>
|
||||
/// <description>D:\huggingface\bge-micro-v2\onnx\model.onnx</description>
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// <term>EmbeddingVocabPath:</term>
|
||||
/// <description>D:\huggingface\bge-micro-v2\vocab.txt</description>
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task RunEmbeddingAsync()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.Onnx.EmbeddingModelPath); // dotnet user-secrets set "Onnx:EmbeddingModelPath" "<model-file-path>"
|
||||
Assert.NotNull(TestConfiguration.Onnx.EmbeddingVocabPath); // dotnet user-secrets set "Onnx:EmbeddingVocabPath" "<vocab-file-path>"
|
||||
|
||||
Console.WriteLine("\n======= Onnx - Embedding Example ========\n");
|
||||
|
||||
Kernel kernel = Kernel.CreateBuilder()
|
||||
.AddBertOnnxEmbeddingGenerator(TestConfiguration.Onnx.EmbeddingModelPath, TestConfiguration.Onnx.EmbeddingVocabPath)
|
||||
.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings for each chunk.
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Example using the service collection directly to get embeddings
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Configuration example:
|
||||
/// <list type="table">
|
||||
/// <item>
|
||||
/// <term>EmbeddingModelPath:</term>
|
||||
/// <description>D:\huggingface\bge-micro-v2\onnx\model.onnx</description>
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// <term>EmbeddingVocabPath:</term>
|
||||
/// <description>D:\huggingface\bge-micro-v2\vocab.txt</description>
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task RunServiceCollectionEmbeddingAsync()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.Onnx.EmbeddingModelPath); // dotnet user-secrets set "Onnx:EmbeddingModelPath" "<model-file-path>"
|
||||
Assert.NotNull(TestConfiguration.Onnx.EmbeddingVocabPath); // dotnet user-secrets set "Onnx:EmbeddingVocabPath" "<vocab-file-path>"
|
||||
|
||||
Console.WriteLine("\n======= Onnx - Embedding Example ========\n");
|
||||
|
||||
var services = new ServiceCollection()
|
||||
.AddBertOnnxEmbeddingGenerator(TestConfiguration.Onnx.EmbeddingModelPath, TestConfiguration.Onnx.EmbeddingVocabPath);
|
||||
var provider = services.BuildServiceProvider();
|
||||
var embeddingGenerator = provider.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings for each chunk.
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(["John: Hello, how are you?\nRoger: Hey, I'm Roger!"]);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.SemanticKernel;
|
||||
using xRetry;
|
||||
|
||||
#pragma warning disable format // Format item can be simplified
|
||||
#pragma warning disable CA1861 // Avoid constant arrays as arguments
|
||||
|
||||
namespace Memory;
|
||||
|
||||
// The following example shows how to use Semantic Kernel with OpenAI.
|
||||
public class OpenAI_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[RetryFact(typeof(HttpOperationException))]
|
||||
public async Task RunEmbeddingAsync()
|
||||
{
|
||||
Assert.NotNull(TestConfiguration.OpenAI.EmbeddingModelId);
|
||||
Assert.NotNull(TestConfiguration.OpenAI.ApiKey);
|
||||
|
||||
IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
|
||||
kernelBuilder.AddOpenAIEmbeddingGenerator(
|
||||
modelId: TestConfiguration.OpenAI.EmbeddingModelId!,
|
||||
apiKey: TestConfiguration.OpenAI.ApiKey!);
|
||||
Kernel kernel = kernelBuilder.Build();
|
||||
|
||||
var embeddingGenerator = kernel.GetRequiredService<IEmbeddingGenerator<string, Embedding<float>>>();
|
||||
|
||||
// Generate embeddings for the specified text.
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your C#, Python, or Java codebase."]);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings for the provided text");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics;
|
||||
using Microsoft.ML.Tokenizers;
|
||||
using Microsoft.SemanticKernel.Text;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
public class TextChunkerUsage(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
|
||||
|
||||
[Fact]
|
||||
public void RunExample()
|
||||
{
|
||||
Console.WriteLine("=== Text chunking ===");
|
||||
|
||||
var lines = TextChunker.SplitPlainTextLines(Text, 40);
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120);
|
||||
|
||||
WriteParagraphsToConsole(paragraphs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RunExampleWithTokenCounter()
|
||||
{
|
||||
Console.WriteLine("=== Text chunking with a custom token counter ===");
|
||||
|
||||
var sw = new Stopwatch();
|
||||
sw.Start();
|
||||
|
||||
var lines = TextChunker.SplitPlainTextLines(Text, 40, text => s_tokenizer.CountTokens(text));
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 120, tokenCounter: text => s_tokenizer.CountTokens(text));
|
||||
|
||||
sw.Stop();
|
||||
Console.WriteLine($"Elapsed time: {sw.ElapsedMilliseconds} ms");
|
||||
WriteParagraphsToConsole(paragraphs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RunExampleWithHeader()
|
||||
{
|
||||
Console.WriteLine("=== Text chunking with chunk header ===");
|
||||
|
||||
var lines = TextChunker.SplitPlainTextLines(Text, 40);
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, 150, chunkHeader: "DOCUMENT NAME: test.txt\n\n");
|
||||
|
||||
WriteParagraphsToConsole(paragraphs);
|
||||
}
|
||||
|
||||
private void WriteParagraphsToConsole(List<string> paragraphs)
|
||||
{
|
||||
for (var i = 0; i < paragraphs.Count; i++)
|
||||
{
|
||||
Console.WriteLine(paragraphs[i]);
|
||||
|
||||
if (i < paragraphs.Count - 1)
|
||||
{
|
||||
Console.WriteLine("------------------------");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private const string Text = """
|
||||
The city of Venice, located in the northeastern part of Italy,
|
||||
is renowned for its unique geographical features. Built on more than 100 small islands in a lagoon in the
|
||||
Adriatic Sea, it has no roads, just canals including the Grand Canal thoroughfare lined with Renaissance and
|
||||
Gothic palaces. The central square, Piazza San Marco, contains St. Mark's Basilica, which is tiled with Byzantine
|
||||
mosaics, and the Campanile bell tower offering views of the city's red roofs.
|
||||
|
||||
The Amazon Rainforest, also known as Amazonia, is a moist broadleaf tropical rainforest in the Amazon biome that
|
||||
covers most of the Amazon basin of South America. This basin encompasses 7 million square kilometers, of which
|
||||
5.5 million square kilometers are covered by the rainforest. This region includes territory belonging to nine nations
|
||||
and 3.4 million square kilometers of uncontacted tribes. The Amazon represents over half of the planet's remaining
|
||||
rainforests and comprises the largest and most biodiverse tract of tropical rainforest in the world.
|
||||
|
||||
The Great Barrier Reef is the world's largest coral reef system composed of over 2,900 individual reefs and 900 islands
|
||||
stretching for over 2,300 kilometers over an area of approximately 344,400 square kilometers. The reef is located in the
|
||||
Coral Sea, off the coast of Queensland, Australia. The Great Barrier Reef can be seen from outer space and is the world's
|
||||
biggest single structure made by living organisms. This reef structure is composed of and built by billions of tiny organisms,
|
||||
known as coral polyps.
|
||||
""";
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.ClientModel;
|
||||
using Azure.AI.OpenAI;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.ML.Tokenizers;
|
||||
using Microsoft.SemanticKernel.Text;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
public class TextChunkingAndEmbedding(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
private const string EmbeddingModelName = "text-embedding-ada-002";
|
||||
private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel(EmbeddingModelName);
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync()
|
||||
{
|
||||
Console.WriteLine("======== Text Embedding ========");
|
||||
await RunExampleAsync();
|
||||
}
|
||||
|
||||
private async Task RunExampleAsync()
|
||||
{
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new ApiKeyCredential(TestConfiguration.AzureOpenAIEmbeddings.ApiKey))
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator();
|
||||
|
||||
// To demonstrate batching we'll create abnormally small partitions.
|
||||
var lines = TextChunker.SplitPlainTextLines(ChatTranscript, maxTokensPerLine: 10);
|
||||
var paragraphs = TextChunker.SplitPlainTextParagraphs(lines, maxTokensPerParagraph: 25);
|
||||
|
||||
Console.WriteLine($"Split transcript into {paragraphs.Count} paragraphs");
|
||||
|
||||
// Azure OpenAI currently supports input arrays up to 16 for text-embedding-ada-002 (Version 2).
|
||||
// Both require the max input token limit per API request to remain under 8191 for this model.
|
||||
var chunks = paragraphs
|
||||
.ChunkByAggregate(
|
||||
seed: 0,
|
||||
aggregator: (tokenCount, paragraph) => tokenCount + s_tokenizer.CountTokens(paragraph),
|
||||
predicate: (tokenCount, index) => tokenCount < 8191 && index < 16)
|
||||
.ToList();
|
||||
|
||||
Console.WriteLine($"Consolidated paragraphs into {chunks.Count}");
|
||||
|
||||
// Generate embeddings for each chunk.
|
||||
for (var i = 0; i < chunks.Count; i++)
|
||||
{
|
||||
var chunk = chunks[i];
|
||||
var embeddings = await embeddingGenerator.GenerateAsync(chunk);
|
||||
|
||||
Console.WriteLine($"Generated {embeddings.Count} embeddings from chunk {i + 1}");
|
||||
}
|
||||
}
|
||||
|
||||
#region Transcript
|
||||
|
||||
private const string ChatTranscript =
|
||||
@"
|
||||
John: Hello, how are you?
|
||||
Jane: I'm fine, thanks. How are you?
|
||||
John: I'm doing well, writing some example code.
|
||||
Jane: That's great! I'm writing some example code too.
|
||||
John: What are you writing?
|
||||
Jane: I'm writing a chatbot.
|
||||
John: That's cool. I'm writing a chatbot too.
|
||||
Jane: What language are you writing it in?
|
||||
John: I'm writing it in C#.
|
||||
Jane: I'm writing it in Python.
|
||||
John: That's cool. I need to learn Python.
|
||||
Jane: I need to learn C#.
|
||||
John: Can I try out your chatbot?
|
||||
Jane: Sure, here's the link.
|
||||
John: Thanks!
|
||||
Jane: You're welcome.
|
||||
Jane: Look at this poem my chatbot wrote:
|
||||
Jane: Roses are red
|
||||
Jane: Violets are blue
|
||||
Jane: I'm writing a chatbot
|
||||
Jane: What about you?
|
||||
John: That's cool. Let me see if mine will write a poem, too.
|
||||
John: Here's a poem my chatbot wrote:
|
||||
John: The singularity of the universe is a mystery.
|
||||
John: The universe is a mystery.
|
||||
John: The universe is a mystery.
|
||||
John: The universe is a mystery.
|
||||
John: Looks like I need to improve mine, oh well.
|
||||
Jane: You might want to try using a different model.
|
||||
Jane: I'm using the GPT-3 model.
|
||||
John: I'm using the GPT-2 model. That makes sense.
|
||||
John: Here is a new poem after updating the model.
|
||||
John: The universe is a mystery.
|
||||
John: The universe is a mystery.
|
||||
John: The universe is a mystery.
|
||||
John: Yikes, it's really stuck isn't it. Would you help me debug my code?
|
||||
Jane: Sure, what's the problem?
|
||||
John: I'm not sure. I think it's a bug in the code.
|
||||
Jane: I'll take a look.
|
||||
Jane: I think I found the problem.
|
||||
Jane: It looks like you're not passing the right parameters to the model.
|
||||
John: Thanks for the help!
|
||||
Jane: I'm now writing a bot to summarize conversations. I want to make sure it works when the conversation is long.
|
||||
John: So you need to keep talking with me to generate a long conversation?
|
||||
Jane: Yes, that's right.
|
||||
John: Ok, I'll keep talking. What should we talk about?
|
||||
Jane: I don't know, what do you want to talk about?
|
||||
John: I don't know, it's nice how CoPilot is doing most of the talking for us. But it definitely gets stuck sometimes.
|
||||
Jane: I agree, it's nice that CoPilot is doing most of the talking for us.
|
||||
Jane: But it definitely gets stuck sometimes.
|
||||
John: Do you know how long it needs to be?
|
||||
Jane: I think the max length is 1024 tokens. Which is approximately 1024*4= 4096 characters.
|
||||
John: That's a lot of characters.
|
||||
Jane: Yes, it is.
|
||||
John: I'm not sure how much longer I can keep talking.
|
||||
Jane: I think we're almost there. Let me check.
|
||||
Jane: I have some bad news, we're only half way there.
|
||||
John: Oh no, I'm not sure I can keep going. I'm getting tired.
|
||||
Jane: I'm getting tired too.
|
||||
John: Maybe there is a large piece of text we can use to generate a long conversation.
|
||||
Jane: That's a good idea. Let me see if I can find one. Maybe Lorem Ipsum?
|
||||
John: Yeah, that's a good idea.
|
||||
Jane: I found a Lorem Ipsum generator.
|
||||
Jane: Here's a 4096 character Lorem Ipsum text:
|
||||
Jane: Lorem ipsum dolor sit amet, con
|
||||
Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam
|
||||
Jane: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed euismod, nunc sit amet aliquam
|
||||
Jane: Darn, it's just repeating stuff now.
|
||||
John: I think we're done.
|
||||
Jane: We're not though! We need like 1500 more characters.
|
||||
John: Oh Cananda, our home and native land.
|
||||
Jane: True patriot love in all thy sons command.
|
||||
John: With glowing hearts we see thee rise.
|
||||
Jane: The True North strong and free.
|
||||
John: From far and wide, O Canada, we stand on guard for thee.
|
||||
Jane: God keep our land glorious and free.
|
||||
John: O Canada, we stand on guard for thee.
|
||||
Jane: O Canada, we stand on guard for thee.
|
||||
Jane: That was fun, thank you. Let me check now.
|
||||
Jane: I think we need about 600 more characters.
|
||||
John: Oh say can you see?
|
||||
Jane: By the dawn's early light.
|
||||
John: What so proudly we hailed.
|
||||
Jane: At the twilight's last gleaming.
|
||||
John: Whose broad stripes and bright stars.
|
||||
Jane: Through the perilous fight.
|
||||
John: O'er the ramparts we watched.
|
||||
Jane: Were so gallantly streaming.
|
||||
John: And the rockets' red glare.
|
||||
Jane: The bombs bursting in air.
|
||||
John: Gave proof through the night.
|
||||
Jane: That our flag was still there.
|
||||
John: Oh say does that star-spangled banner yet wave.
|
||||
Jane: O'er the land of the free.
|
||||
John: And the home of the brave.
|
||||
Jane: Are you a Seattle Kraken Fan?
|
||||
John: Yes, I am. I love going to the games.
|
||||
Jane: I'm a Seattle Kraken Fan too. Who is your favorite player?
|
||||
John: I like watching all the players, but I think my favorite is Matty Beniers.
|
||||
Jane: Yeah, he's a great player. I like watching him too. I also like watching Jaden Schwartz.
|
||||
John: Adam Larsson is another good one. The big cat!
|
||||
Jane: WE MADE IT! It's long enough. Thank you!
|
||||
John: You're welcome. I'm glad we could help. Goodbye!
|
||||
Jane: Goodbye!
|
||||
";
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Data;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for <see cref="VectorStore"/> which allow:
|
||||
/// 1. Creating an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings.
|
||||
/// </summary>
|
||||
internal static class VectorStoreExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Delegate to create a record from a string.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Type of the record key.</typeparam>
|
||||
/// <typeparam name="TRecord">Type of the record.</typeparam>
|
||||
internal delegate TRecord CreateRecordFromString<TKey, TRecord>(string text, ReadOnlyMemory<float> vector) where TKey : notnull;
|
||||
|
||||
/// <summary>
|
||||
/// Delegate to create a record from a <see cref="TextSearchResult"/>.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Type of the record key.</typeparam>
|
||||
/// <typeparam name="TRecord">Type of the record.</typeparam>
|
||||
internal delegate TRecord CreateRecordFromTextSearchResult<TKey, TRecord>(TextSearchResult searchResult, ReadOnlyMemory<float> vector) where TKey : notnull;
|
||||
|
||||
/// <summary>
|
||||
/// Create a <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings by:
|
||||
/// 1. Getting an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/>
|
||||
/// 2. Generating embeddings for each string.
|
||||
/// 3. Creating a record with a valid key for each string and it's embedding.
|
||||
/// 4. Insert the records into the collection.
|
||||
/// </summary>
|
||||
/// <param name="vectorStore">Instance of <see cref="VectorStore"/> used to created the collection.</param>
|
||||
/// <param name="collectionName">The collection name.</param>
|
||||
/// <param name="entries">A list of strings.</param>
|
||||
/// <param name="embeddingGenerator">An embedding generator.</param>
|
||||
/// <param name="createRecord">A delegate which can create a record with a valid key for each string and it's embedding.</param>
|
||||
internal static async Task<VectorStoreCollection<TKey, TRecord>> CreateCollectionFromListAsync<TKey, TRecord>(
|
||||
this VectorStore vectorStore,
|
||||
string collectionName,
|
||||
string[] entries,
|
||||
IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
|
||||
CreateRecordFromString<TKey, TRecord> createRecord)
|
||||
where TKey : notnull
|
||||
where TRecord : class
|
||||
{
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<TKey, TRecord>(collectionName);
|
||||
await collection.EnsureCollectionExistsAsync().ConfigureAwait(false);
|
||||
|
||||
// Create records and generate embeddings for them.
|
||||
var tasks = entries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
var record = createRecord(entry, (await embeddingGenerator.GenerateAsync(entry).ConfigureAwait(false)).Vector);
|
||||
await collection.UpsertAsync(record).ConfigureAwait(false);
|
||||
}));
|
||||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
||||
|
||||
return collection;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a <see cref="VectorStoreCollection{TKey, TRecord}"/> from a list of strings by:
|
||||
/// 1. Getting an instance of <see cref="VectorStoreCollection{TKey, TRecord}"/>
|
||||
/// 2. Generating embeddings for each string.
|
||||
/// 3. Creating a record with a valid key for each string and it's embedding.
|
||||
/// 4. Insert the records into the collection.
|
||||
/// </summary>
|
||||
/// <param name="vectorStore">Instance of <see cref="VectorStore"/> used to created the collection.</param>
|
||||
/// <param name="collectionName">The collection name.</param>
|
||||
/// <param name="searchResults">A list of <see cref="TextSearchResult" />s.</param>
|
||||
/// <param name="embeddingGenerator">An embedding generator service.</param>
|
||||
/// <param name="createRecord">A delegate which can create a record with a valid key for each string and it's embedding.</param>
|
||||
internal static async Task<VectorStoreCollection<TKey, TRecord>> CreateCollectionFromTextSearchResultsAsync<TKey, TRecord>(
|
||||
this VectorStore vectorStore,
|
||||
string collectionName,
|
||||
IList<TextSearchResult> searchResults,
|
||||
IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
|
||||
CreateRecordFromTextSearchResult<TKey, TRecord> createRecord)
|
||||
where TKey : notnull
|
||||
where TRecord : class
|
||||
{
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<TKey, TRecord>(collectionName);
|
||||
await collection.EnsureCollectionExistsAsync().ConfigureAwait(false);
|
||||
|
||||
// Create records and generate embeddings for them.
|
||||
var tasks = searchResults.Select(searchResult => Task.Run(async () =>
|
||||
{
|
||||
var record = createRecord(searchResult, (await embeddingGenerator.GenerateAsync(searchResult.Value!).ConfigureAwait(false)).Vector);
|
||||
await collection.UpsertAsync(record).ConfigureAwait(false);
|
||||
}));
|
||||
await Task.WhenAll(tasks).ConfigureAwait(false);
|
||||
|
||||
return collection;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Docker.DotNet;
|
||||
using Docker.DotNet.Models;
|
||||
|
||||
namespace Memory.VectorStoreFixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Helper class that creates and deletes containers for the vector store examples.
|
||||
/// </summary>
|
||||
internal static class VectorStoreInfra
|
||||
{
|
||||
/// <summary>
|
||||
/// Setup the postgres pgvector container by pulling the image and running it.
|
||||
/// </summary>
|
||||
/// <param name="client">The docker client to create the container with.</param>
|
||||
/// <returns>The id of the container.</returns>
|
||||
public static async Task<string> SetupPostgresContainerAsync(DockerClient client)
|
||||
{
|
||||
await client.Images.CreateImageAsync(
|
||||
new ImagesCreateParameters
|
||||
{
|
||||
FromImage = "pgvector/pgvector",
|
||||
Tag = "pg16",
|
||||
},
|
||||
null,
|
||||
new Progress<JSONMessage>());
|
||||
|
||||
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
|
||||
{
|
||||
Image = "pgvector/pgvector:pg16",
|
||||
HostConfig = new HostConfig()
|
||||
{
|
||||
PortBindings = new Dictionary<string, IList<PortBinding>>
|
||||
{
|
||||
{"5432", new List<PortBinding> {new() {HostPort = "5432" } }},
|
||||
},
|
||||
PublishAllPorts = true
|
||||
},
|
||||
ExposedPorts = new Dictionary<string, EmptyStruct>
|
||||
{
|
||||
{ "5432", default },
|
||||
},
|
||||
Env =
|
||||
[
|
||||
"POSTGRES_USER=postgres",
|
||||
"POSTGRES_PASSWORD=example",
|
||||
],
|
||||
});
|
||||
|
||||
await client.Containers.StartContainerAsync(
|
||||
container.ID,
|
||||
new ContainerStartParameters());
|
||||
|
||||
return container.ID;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Setup the qdrant container by pulling the image and running it.
|
||||
/// </summary>
|
||||
/// <param name="client">The docker client to create the container with.</param>
|
||||
/// <returns>The id of the container.</returns>
|
||||
public static async Task<string> SetupQdrantContainerAsync(DockerClient client)
|
||||
{
|
||||
await client.Images.CreateImageAsync(
|
||||
new ImagesCreateParameters
|
||||
{
|
||||
FromImage = "qdrant/qdrant",
|
||||
Tag = "latest",
|
||||
},
|
||||
null,
|
||||
new Progress<JSONMessage>());
|
||||
|
||||
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
|
||||
{
|
||||
Image = "qdrant/qdrant",
|
||||
HostConfig = new HostConfig()
|
||||
{
|
||||
PortBindings = new Dictionary<string, IList<PortBinding>>
|
||||
{
|
||||
{"6333", new List<PortBinding> {new() {HostPort = "6333" } }},
|
||||
{"6334", new List<PortBinding> {new() {HostPort = "6334" } }}
|
||||
},
|
||||
PublishAllPorts = true
|
||||
},
|
||||
ExposedPorts = new Dictionary<string, EmptyStruct>
|
||||
{
|
||||
{ "6333", default },
|
||||
{ "6334", default }
|
||||
},
|
||||
});
|
||||
|
||||
await client.Containers.StartContainerAsync(
|
||||
container.ID,
|
||||
new ContainerStartParameters());
|
||||
|
||||
return container.ID;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Setup the redis container by pulling the image and running it.
|
||||
/// </summary>
|
||||
/// <param name="client">The docker client to create the container with.</param>
|
||||
/// <returns>The id of the container.</returns>
|
||||
public static async Task<string> SetupRedisContainerAsync(DockerClient client)
|
||||
{
|
||||
await client.Images.CreateImageAsync(
|
||||
new ImagesCreateParameters
|
||||
{
|
||||
FromImage = "redis/redis-stack",
|
||||
Tag = "latest",
|
||||
},
|
||||
null,
|
||||
new Progress<JSONMessage>());
|
||||
|
||||
var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
|
||||
{
|
||||
Image = "redis/redis-stack",
|
||||
HostConfig = new HostConfig()
|
||||
{
|
||||
PortBindings = new Dictionary<string, IList<PortBinding>>
|
||||
{
|
||||
{"6379", new List<PortBinding> {new() {HostPort = "6379"}}},
|
||||
{"8001", new List<PortBinding> {new() {HostPort = "8001"}}}
|
||||
},
|
||||
PublishAllPorts = true
|
||||
},
|
||||
ExposedPorts = new Dictionary<string, EmptyStruct>
|
||||
{
|
||||
{ "6379", default },
|
||||
{ "8001", default }
|
||||
},
|
||||
});
|
||||
|
||||
await client.Containers.StartContainerAsync(
|
||||
container.ID,
|
||||
new ContainerStartParameters());
|
||||
|
||||
return container.ID;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stop and delete the container with the specified id.
|
||||
/// </summary>
|
||||
/// <param name="client">The docker client to delete the container in.</param>
|
||||
/// <param name="containerId">The id of the container to delete.</param>
|
||||
/// <returns>An async task.</returns>
|
||||
public static async Task DeleteContainerAsync(DockerClient client, string containerId)
|
||||
{
|
||||
await client.Containers.StopContainerAsync(containerId, new ContainerStopParameters());
|
||||
await client.Containers.RemoveContainerAsync(containerId, new ContainerRemoveParameters());
|
||||
}
|
||||
}
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Docker.DotNet;
|
||||
using Npgsql;
|
||||
|
||||
namespace Memory.VectorStoreFixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Fixture to use for creating a Postgres container before tests and delete it after tests.
|
||||
/// </summary>
|
||||
public class VectorStorePostgresContainerFixture : IAsyncLifetime
|
||||
{
|
||||
private DockerClient? _dockerClient;
|
||||
private string? _postgresContainerId;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
}
|
||||
|
||||
public async Task ManualInitializeAsync()
|
||||
{
|
||||
if (this._postgresContainerId == null)
|
||||
{
|
||||
// Connect to docker and start the docker container.
|
||||
using var dockerClientConfiguration = new DockerClientConfiguration();
|
||||
this._dockerClient = dockerClientConfiguration.CreateClient();
|
||||
this._postgresContainerId = await VectorStoreInfra.SetupPostgresContainerAsync(this._dockerClient);
|
||||
|
||||
// Delay until the Postgres server is ready.
|
||||
var connectionString = "Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;";
|
||||
var succeeded = false;
|
||||
var attemptCount = 0;
|
||||
while (!succeeded && attemptCount++ < 10)
|
||||
{
|
||||
try
|
||||
{
|
||||
NpgsqlDataSourceBuilder dataSourceBuilder = new(connectionString);
|
||||
dataSourceBuilder.UseVector();
|
||||
using var dataSource = dataSourceBuilder.Build();
|
||||
NpgsqlConnection connection = await dataSource.OpenConnectionAsync().ConfigureAwait(false);
|
||||
|
||||
await using (connection)
|
||||
{
|
||||
// Create extension vector if it doesn't exist
|
||||
await using (NpgsqlCommand command = new("CREATE EXTENSION IF NOT EXISTS vector", connection))
|
||||
{
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (this._dockerClient != null && this._postgresContainerId != null)
|
||||
{
|
||||
// Delete docker container.
|
||||
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._postgresContainerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Docker.DotNet;
|
||||
using Qdrant.Client;
|
||||
|
||||
namespace Memory.VectorStoreFixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Fixture to use for creating a Qdrant container before tests and delete it after tests.
|
||||
/// </summary>
|
||||
public class VectorStoreQdrantContainerFixture : IAsyncLifetime
|
||||
{
|
||||
private DockerClient? _dockerClient;
|
||||
private string? _qdrantContainerId;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
}
|
||||
|
||||
public async Task ManualInitializeAsync()
|
||||
{
|
||||
if (this._qdrantContainerId == null)
|
||||
{
|
||||
// Connect to docker and start the docker container.
|
||||
using var dockerClientConfiguration = new DockerClientConfiguration();
|
||||
this._dockerClient = dockerClientConfiguration.CreateClient();
|
||||
this._qdrantContainerId = await VectorStoreInfra.SetupQdrantContainerAsync(this._dockerClient);
|
||||
|
||||
// Delay until the Qdrant server is ready.
|
||||
var qdrantClient = new QdrantClient("localhost");
|
||||
var succeeded = false;
|
||||
var attemptCount = 0;
|
||||
while (!succeeded && attemptCount++ < 10)
|
||||
{
|
||||
try
|
||||
{
|
||||
await qdrantClient.ListCollectionsAsync();
|
||||
succeeded = true;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (this._dockerClient != null && this._qdrantContainerId != null)
|
||||
{
|
||||
// Delete docker container.
|
||||
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._qdrantContainerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Docker.DotNet;
|
||||
|
||||
namespace Memory.VectorStoreFixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Fixture to use for creating a Redis container before tests and delete it after tests.
|
||||
/// </summary>
|
||||
public class VectorStoreRedisContainerFixture : IAsyncLifetime
|
||||
{
|
||||
private DockerClient? _dockerClient;
|
||||
private string? _redisContainerId;
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
}
|
||||
|
||||
public async Task ManualInitializeAsync()
|
||||
{
|
||||
if (this._redisContainerId == null)
|
||||
{
|
||||
// Connect to docker and start the docker container.
|
||||
using var dockerClientConfiguration = new DockerClientConfiguration();
|
||||
this._dockerClient = dockerClientConfiguration.CreateClient();
|
||||
this._redisContainerId = await VectorStoreInfra.SetupRedisContainerAsync(this._dockerClient);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (this._dockerClient != null && this._redisContainerId != null)
|
||||
{
|
||||
// Delete docker container.
|
||||
await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._redisContainerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Memory.VectorStoreLangchainInterop;
|
||||
|
||||
/// <summary>
|
||||
/// Data model class that matches the data model used by Langchain.
|
||||
/// This data model is not decorated with vector store attributes since instead
|
||||
/// a different record definition is used with each vector store implementation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
|
||||
/// </remarks>
|
||||
public class LangchainDocument<TKey>
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier of the record.
|
||||
/// </summary>
|
||||
public TKey Key { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The text content for which embeddings have been generated.
|
||||
/// </summary>
|
||||
public string Content { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The source of the content. E.g. where to find the original content.
|
||||
/// </summary>
|
||||
public string Source { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The embedding for the <see cref="Content"/>.
|
||||
/// </summary>
|
||||
public ReadOnlyMemory<float> Embedding { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Pinecone;
|
||||
using Pinecone;
|
||||
|
||||
namespace Memory.VectorStoreLangchainInterop;
|
||||
|
||||
/// <summary>
|
||||
/// Contains a factory method that can be used to create a Pinecone vector store that is compatible with datasets ingested using Langchain.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
|
||||
/// </remarks>
|
||||
public static class PineconeFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Record definition that matches the storage format used by Langchain for Pinecone.
|
||||
/// </summary>
|
||||
private static readonly VectorStoreCollectionDefinition s_definition = new()
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("Key", typeof(string)),
|
||||
new VectorStoreDataProperty("Content", typeof(string)) { StorageName = "text" },
|
||||
new VectorStoreDataProperty("Source", typeof(string)) { StorageName = "source" },
|
||||
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 1536) { StorageName = "embedding" }
|
||||
]
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Create a new Pinecone-backed <see cref="VectorStore"/> that can be used to read data that was ingested using Langchain.
|
||||
/// </summary>
|
||||
/// <param name="pineconeClient">Pinecone client that can be used to manage the collections and points in a Pinecone store.</param>
|
||||
/// <returns>The <see cref="VectorStore"/>.</returns>
|
||||
public static VectorStore CreatePineconeLangchainInteropVectorStore(PineconeClient pineconeClient)
|
||||
=> new PineconeLangchainInteropVectorStore(new PineconeVectorStore(pineconeClient), pineconeClient);
|
||||
|
||||
private sealed class PineconeLangchainInteropVectorStore(
|
||||
VectorStore innerStore,
|
||||
PineconeClient pineconeClient)
|
||||
: VectorStore
|
||||
{
|
||||
private readonly PineconeClient _pineconeClient = pineconeClient;
|
||||
|
||||
public override VectorStoreCollection<TKey, TRecord> GetCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition? definition = null)
|
||||
{
|
||||
if (typeof(TKey) != typeof(string) || typeof(TRecord) != typeof(LangchainDocument<string>))
|
||||
{
|
||||
throw new NotSupportedException("This VectorStore is only usable with string keys and LangchainDocument<string> record types");
|
||||
}
|
||||
|
||||
// Create a Pinecone collection and pass in our custom record definition that matches
|
||||
// the schema used by Langchain so that the default mapper can use the storage names
|
||||
// in it, to map to the storage scheme.
|
||||
return (new PineconeCollection<TKey, TRecord>(
|
||||
_pineconeClient,
|
||||
name,
|
||||
new()
|
||||
{
|
||||
Definition = s_definition
|
||||
}) as VectorStoreCollection<TKey, TRecord>)!;
|
||||
}
|
||||
|
||||
public override VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null)
|
||||
{
|
||||
// Create a Pinecone collection and pass in our custom record definition that matches
|
||||
// the schema used by Langchain so that the default mapper can use the storage names
|
||||
// in it, to map to the storage scheme.
|
||||
return new PineconeDynamicCollection(
|
||||
_pineconeClient,
|
||||
name,
|
||||
new()
|
||||
{
|
||||
Definition = s_definition
|
||||
});
|
||||
}
|
||||
|
||||
public override object? GetService(Type serviceType, object? serviceKey = null) => innerStore.GetService(serviceType, serviceKey);
|
||||
|
||||
public override IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default) => innerStore.ListCollectionNamesAsync(cancellationToken);
|
||||
|
||||
public override Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => innerStore.CollectionExistsAsync(name, cancellationToken);
|
||||
|
||||
public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => innerStore.EnsureCollectionDeletedAsync(name, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Redis;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Memory.VectorStoreLangchainInterop;
|
||||
|
||||
/// <summary>
|
||||
/// Contains a factory method that can be used to create a Redis vector store that is compatible with datasets ingested using Langchain.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This class is used with the <see cref="VectorStore_Langchain_Interop"/> sample.
|
||||
/// </remarks>
|
||||
public static class RedisFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Record definition that matches the storage format used by Langchain for Redis.
|
||||
/// </summary>
|
||||
private static readonly VectorStoreCollectionDefinition s_definition = new()
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("Key", typeof(string)),
|
||||
new VectorStoreDataProperty("Content", typeof(string)) { StorageName = "text" },
|
||||
new VectorStoreDataProperty("Source", typeof(string)) { StorageName = "source" },
|
||||
new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory<float>), 1536) { StorageName = "embedding" }
|
||||
]
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Create a new Redis-backed <see cref="VectorStore"/> that can be used to read data that was ingested using Langchain.
|
||||
/// </summary>
|
||||
/// <param name="database">The redis database to read/write from.</param>
|
||||
/// <returns>The <see cref="VectorStore"/>.</returns>
|
||||
public static VectorStore CreateRedisLangchainInteropVectorStore(IDatabase database)
|
||||
=> new RedisLangchainInteropVectorStore(new RedisVectorStore(database), database);
|
||||
|
||||
private sealed class RedisLangchainInteropVectorStore(
|
||||
VectorStore innerStore,
|
||||
IDatabase database)
|
||||
: VectorStore
|
||||
{
|
||||
private readonly IDatabase _database = database;
|
||||
|
||||
public override VectorStoreCollection<TKey, TRecord> GetCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition? definition = null)
|
||||
{
|
||||
if (typeof(TKey) != typeof(string) || typeof(TRecord) != typeof(LangchainDocument<string>))
|
||||
{
|
||||
throw new NotSupportedException("This VectorStore is only usable with string keys and LangchainDocument<string> record types");
|
||||
}
|
||||
|
||||
// Create a hash set collection, since Langchain uses redis hashes for storing records.
|
||||
// Also pass in our custom record definition that matches the schema used by Langchain
|
||||
// so that the default mapper can use the storage names in it, to map to the storage
|
||||
// scheme.
|
||||
return (new RedisHashSetCollection<TKey, TRecord>(
|
||||
_database,
|
||||
name,
|
||||
new()
|
||||
{
|
||||
Definition = s_definition
|
||||
}) as VectorStoreCollection<TKey, TRecord>)!;
|
||||
}
|
||||
|
||||
public override VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null)
|
||||
{
|
||||
// Create a hash set collection, since Langchain uses redis hashes for storing records.
|
||||
// Also pass in our custom record definition that matches the schema used by Langchain
|
||||
// so that the default mapper can use the storage names in it, to map to the storage
|
||||
// scheme.
|
||||
return new RedisHashSetDynamicCollection(
|
||||
_database,
|
||||
name,
|
||||
new()
|
||||
{
|
||||
Definition = s_definition
|
||||
});
|
||||
}
|
||||
|
||||
public override object? GetService(Type serviceType, object? serviceKey = null) => innerStore.GetService(serviceType, serviceKey);
|
||||
|
||||
public override IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default) => innerStore.ListCollectionNamesAsync(cancellationToken);
|
||||
|
||||
public override Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => innerStore.CollectionExistsAsync(name, cancellationToken);
|
||||
|
||||
public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => innerStore.EnsureCollectionDeletedAsync(name, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Azure;
|
||||
using Azure.Search.Documents.Indexes;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how use the VectorStore abstractions to consume data from an Azure AI Search data store,
|
||||
/// that was created using the MemoryStore abstractions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
|
||||
/// e.g. it only supports a single fixed schema and does not allow search filtering.
|
||||
/// To provide more flexibility, the Vector Store abstraction has been introduced.
|
||||
///
|
||||
/// To run this sample, you need an instance of Azure AI Search available and configured.
|
||||
/// dotnet user-secrets set "AzureAISearch:Endpoint" "https://myazureaisearchinstance.search.windows.net"
|
||||
/// dotnet user-secrets set "AzureAISearch:ApiKey" "samplesecret"
|
||||
/// </remarks>
|
||||
public class VectorStore_ConsumeFromMemoryStore_AzureAISearch(ITestOutputHelper output) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
private const int VectorSize = 1536;
|
||||
private static readonly JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeExampleAsync()
|
||||
{
|
||||
// Construct a VectorStore.
|
||||
var vectorStore = new AzureAISearchVectorStore(new SearchIndexClient(
|
||||
new Uri(TestConfiguration.AzureAISearch.Endpoint),
|
||||
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey)));
|
||||
|
||||
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
|
||||
var collection = vectorStore.GetCollection<string, VectorStoreRecord>("memorystorecollection");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Show that the data can be read using the VectorStore abstraction.
|
||||
// Note that AzureAISearchMemoryStore converts all keys to base64
|
||||
// strings on upload so we need to encode the ids here before doing a get.
|
||||
var record1 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("11111111-1111-1111-1111-111111111111")));
|
||||
var record2 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("22222222-2222-2222-2222-222222222222")));
|
||||
var record3 = await collection.GetAsync(Convert.ToBase64String(Encoding.UTF8.GetBytes("33333333-3333-3333-3333-333333333333")), new() { IncludeVectors = true });
|
||||
|
||||
Console.WriteLine($"Record 1: {JsonSerializer.Serialize(record1, s_consoleFormatting)}");
|
||||
Console.WriteLine($"Record 2: {JsonSerializer.Serialize(record2, s_consoleFormatting)}");
|
||||
Console.WriteLine($"Record 3: {JsonSerializer.Serialize(record3, s_consoleFormatting)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data model with Vector Store attributes that matches the storage representation of
|
||||
/// <see cref="MemoryRecord"/> objects as created by <c>AzureAISearchMemoryStore</c>.
|
||||
/// </summary>
|
||||
private sealed class VectorStoreRecord
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public string Id { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Description { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Text { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public bool IsReference { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string ExternalSourceName { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string AdditionalMetadata { get; set; }
|
||||
|
||||
[VectorStoreVector(VectorSize)]
|
||||
public ReadOnlyMemory<float> Embedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text.Json;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Qdrant;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using Qdrant.Client;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how use the VectorStore abstractions to consume data from a Qdrant data store,
|
||||
/// that was created using the MemoryStore abstractions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
|
||||
/// e.g. it only supports a single fixed schema and does not allow search filtering.
|
||||
/// To provide more flexibility, the Vector Store abstraction has been introduced.
|
||||
///
|
||||
/// To run this sample, you need a local instance of Docker running, since the associated fixture
|
||||
/// will try and start a Qdrant container in the local docker instance to run against.
|
||||
/// </remarks>
|
||||
public class VectorStore_ConsumeFromMemoryStore_Qdrant(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
private const int VectorSize = 1536;
|
||||
private static readonly JsonSerializerOptions s_consoleFormatting = new() { WriteIndented = true };
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeExampleAsync()
|
||||
{
|
||||
// Setup the supporting infra and embedding generation.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
|
||||
// Construct a VectorStore.
|
||||
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
|
||||
|
||||
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction
|
||||
var collection = vectorStore.GetCollection<Guid, VectorStoreRecord>("memorystorecollection");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Show that the data can be read using the VectorStore abstraction.
|
||||
var record1 = await collection.GetAsync(new Guid("11111111-1111-1111-1111-111111111111"));
|
||||
var record2 = await collection.GetAsync(new Guid("22222222-2222-2222-2222-222222222222"));
|
||||
var record3 = await collection.GetAsync(new Guid("33333333-3333-3333-3333-333333333333"), new() { IncludeVectors = true });
|
||||
|
||||
Console.WriteLine($"Record 1: {JsonSerializer.Serialize(record1, s_consoleFormatting)}");
|
||||
Console.WriteLine($"Record 2: {JsonSerializer.Serialize(record2, s_consoleFormatting)}");
|
||||
Console.WriteLine($"Record 3: {JsonSerializer.Serialize(record3, s_consoleFormatting)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data model with Vector Store attributes that matches the storage representation of
|
||||
/// <see cref="MemoryRecord"/> objects as created by <c>QdrantMemoryStore</c>.
|
||||
/// </summary>
|
||||
private sealed class VectorStoreRecord
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public Guid Key { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "id")]
|
||||
public string Id { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "description")]
|
||||
public string Description { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "text")]
|
||||
public string Text { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "is_reference")]
|
||||
public bool IsReference { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "external_source_name")]
|
||||
public string ExternalSourceName { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "additional_metadata")]
|
||||
public string AdditionalMetadata { get; set; }
|
||||
|
||||
[VectorStoreVector(VectorSize)]
|
||||
public ReadOnlyMemory<float> Embedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Redis;
|
||||
using Microsoft.SemanticKernel.Memory;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how use the VectorStore abstractions to consume data from a Redis data store,
|
||||
/// that was created using the MemoryStore abstractions.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The IMemoryStore abstraction has limitations that constrain its use in many scenarios
|
||||
/// e.g. it only supports a single fixed schema and does not allow search filtering.
|
||||
/// To provide more flexibility, the Vector Store abstraction has been introduced.
|
||||
///
|
||||
/// To run this sample, you need a local instance of Docker running, since the associated fixture
|
||||
/// will try and start a Redis container in the local docker instance to run against.
|
||||
/// </remarks>
|
||||
public class VectorStore_ConsumeFromMemoryStore_Redis(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>
|
||||
{
|
||||
private const int VectorSize = 1536;
|
||||
private const string MemoryStoreCollectionName = "memorystorecollection";
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeExampleAsync()
|
||||
{
|
||||
// Setup the supporting infra and embedding generation.
|
||||
await redisFixture.ManualInitializeAsync();
|
||||
|
||||
// Use the VectorStore abstraction to connect to an existing collection which was previously created via the IMemoryStore abstraction.
|
||||
// Note that we use HashSet since the legacy memory store uses hashes to store memory records.
|
||||
var vectorStore = new RedisVectorStore(
|
||||
ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(),
|
||||
new() { StorageType = RedisStorageType.HashSet });
|
||||
|
||||
// Connect to the same collection using the VectorStore abstraction.
|
||||
var collection = vectorStore.GetCollection<string, VectorStoreRecord>(MemoryStoreCollectionName);
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Show that the data can be read using the VectorStore abstraction.
|
||||
var record1 = await collection.GetAsync("11111111-1111-1111-1111-111111111111");
|
||||
var record2 = await collection.GetAsync("22222222-2222-2222-2222-222222222222");
|
||||
var record3 = await collection.GetAsync("33333333-3333-3333-3333-333333333333", new() { IncludeVectors = true });
|
||||
|
||||
Console.WriteLine($"Record 1: Key: {record1!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record1.Timestamp)} Metadata: {record1.Metadata} Embedding {record1.Embedding}");
|
||||
Console.WriteLine($"Record 2: Key: {record2!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record2.Timestamp)} Metadata: {record2.Metadata} Embedding {record2.Embedding}");
|
||||
Console.WriteLine($"Record 3: Key: {record3!.Key} Timestamp: {DateTimeOffset.FromUnixTimeMilliseconds(record3.Timestamp)} Metadata: {record3.Metadata} Embedding {record3.Embedding}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data model with Vector Store attributes that matches the storage representation of
|
||||
/// <see cref="MemoryRecord"/> objects as created by <c>RedisMemoryStore</c>.
|
||||
/// </summary>
|
||||
private sealed class VectorStoreRecord
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public string Key { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "metadata")]
|
||||
public string Metadata { get; set; }
|
||||
|
||||
[VectorStoreData(StorageName = "timestamp")]
|
||||
public long Timestamp { get; set; }
|
||||
|
||||
[VectorStoreVector(VectorSize, StorageName = "embedding")]
|
||||
public ReadOnlyMemory<float> Embedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text.Json;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
using Microsoft.SemanticKernel.Connectors.Qdrant;
|
||||
using Microsoft.SemanticKernel.Connectors.Redis;
|
||||
using Qdrant.Client;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to ingest data into a vector store using <see cref="RedisVectorStore"/>, <see cref="QdrantVectorStore"/> or <see cref="InMemoryVectorStore"/>.
|
||||
/// Since Redis and InMemory supports string keys and Qdrant supports ulong or Guid keys, this example also shows how you can have common code
|
||||
/// that works with both types of keys by using a generic key generator function.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Register a vector store and embedding generator with the DI container.
|
||||
/// 2. Register a class (DataIngestor) with the DI container that uses the vector store and embedding generator to ingest data.
|
||||
/// 3. Ingest some data into the vector store.
|
||||
/// 4. Read the data back from the vector store.
|
||||
///
|
||||
/// For some databases in this sample (Redis & Qdrant), you need a local instance of Docker running, since the associated fixtures will try and start containers in the local docker instance to run against.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
public class VectorStore_DataIngestion_MultiStore(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>, IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
/// <summary>
|
||||
/// Example with dependency injection.
|
||||
/// </summary>
|
||||
/// <param name="databaseType">The type of database to run the example for.</param>
|
||||
[Theory]
|
||||
[InlineData("Redis")]
|
||||
[InlineData("Qdrant")]
|
||||
[InlineData("InMemory")]
|
||||
public async Task ExampleWithDIAsync(string databaseType)
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Register the chosen vector store with the DI container and initialize docker containers via the fixtures where needed.
|
||||
if (databaseType == "Redis")
|
||||
{
|
||||
await redisFixture.ManualInitializeAsync();
|
||||
kernelBuilder.Services.AddRedisVectorStore("localhost:6379");
|
||||
}
|
||||
else if (databaseType == "Qdrant")
|
||||
{
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
kernelBuilder.Services.AddQdrantVectorStore("localhost", https: false);
|
||||
}
|
||||
else if (databaseType == "InMemory")
|
||||
{
|
||||
kernelBuilder.Services.AddInMemoryVectorStore();
|
||||
}
|
||||
|
||||
// Register the DataIngestor with the DI container.
|
||||
kernelBuilder.Services.AddTransient<DataIngestor>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a DataIngestor object using the DI container.
|
||||
var dataIngestor = kernel.GetRequiredService<DataIngestor>();
|
||||
|
||||
// Invoke the data ingestor using an appropriate key generator function for each database type.
|
||||
// Redis and InMemory supports string keys, while Qdrant supports ulong or Guid keys, so we use a different key generator for each key type.
|
||||
if (databaseType is "Redis" or "InMemory")
|
||||
{
|
||||
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid().ToString());
|
||||
}
|
||||
else if (databaseType == "Qdrant")
|
||||
{
|
||||
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Example without dependency injection.
|
||||
/// </summary>
|
||||
/// <param name="databaseType">The type of database to run the example for.</param>
|
||||
[Theory]
|
||||
[InlineData("Redis")]
|
||||
[InlineData("Qdrant")]
|
||||
[InlineData("InMemory")]
|
||||
public async Task ExampleWithoutDIAsync(string databaseType)
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct the chosen vector store and initialize docker containers via the fixtures where needed.
|
||||
VectorStore vectorStore;
|
||||
if (databaseType == "Redis")
|
||||
{
|
||||
await redisFixture.ManualInitializeAsync();
|
||||
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
|
||||
vectorStore = new RedisVectorStore(database);
|
||||
}
|
||||
else if (databaseType == "Qdrant")
|
||||
{
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
var qdrantClient = new QdrantClient("localhost", https: false);
|
||||
vectorStore = new QdrantVectorStore(qdrantClient, ownsClient: true);
|
||||
}
|
||||
else if (databaseType == "InMemory")
|
||||
{
|
||||
vectorStore = new InMemoryVectorStore();
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new ArgumentException("Invalid database type.");
|
||||
}
|
||||
|
||||
// Create the DataIngestor.
|
||||
var dataIngestor = new DataIngestor(vectorStore, embeddingGenerator);
|
||||
|
||||
// Invoke the data ingestor using an appropriate key generator function for each database type.
|
||||
// Redis and InMemory supports string keys, while Qdrant supports ulong or Guid keys, so we use a different key generator for each key type.
|
||||
if (databaseType is "Redis" or "InMemory")
|
||||
{
|
||||
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid().ToString());
|
||||
}
|
||||
else if (databaseType == "Qdrant")
|
||||
{
|
||||
await this.UpsertDataAndReadFromVectorStoreAsync(dataIngestor, () => Guid.NewGuid());
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpsertDataAndReadFromVectorStoreAsync<TKey>(DataIngestor dataIngestor, Func<TKey> uniqueKeyGenerator)
|
||||
where TKey : notnull
|
||||
{
|
||||
// Ingest some data into the vector store.
|
||||
var upsertedKeys = await dataIngestor.ImportDataAsync(uniqueKeyGenerator);
|
||||
|
||||
// Get one of the upserted records.
|
||||
var upsertedRecord = await dataIngestor.GetGlossaryAsync(upsertedKeys.First());
|
||||
|
||||
// Write upserted keys and one of the upserted records to the console.
|
||||
Console.WriteLine($"Upserted keys: {string.Join(", ", upsertedKeys)}");
|
||||
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample class that does ingestion of sample data into a vector store and allows retrieval of data from the vector store.
|
||||
/// </summary>
|
||||
/// <param name="vectorStore">The vector store to ingest data into.</param>
|
||||
/// <param name="embeddingGenerator">Used to generate embeddings for the data being ingested.</param>
|
||||
private sealed class DataIngestor(VectorStore vectorStore, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator)
|
||||
{
|
||||
/// <summary>
|
||||
/// Create some glossary entries and upsert them into the vector store.
|
||||
/// </summary>
|
||||
/// <returns>The keys of the upserted glossary entries.</returns>
|
||||
/// <typeparam name="TKey">The type of the keys in the vector store.</typeparam>
|
||||
public async Task<IEnumerable<TKey>> ImportDataAsync<TKey>(Func<TKey> uniqueKeyGenerator)
|
||||
where TKey : notnull
|
||||
{
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>("skglossary");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateGlossaryEntries(uniqueKeyGenerator).ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection and return their keys.
|
||||
await collection.UpsertAsync(glossaryEntries);
|
||||
|
||||
return glossaryEntries.Select(entry => entry.Key);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get a glossary entry from the vector store.
|
||||
/// </summary>
|
||||
/// <param name="key">The key of the glossary entry to retrieve.</param>
|
||||
/// <returns>The glossary entry.</returns>
|
||||
/// <typeparam name="TKey">The type of the keys in the vector store.</typeparam>
|
||||
public Task<Glossary<TKey>?> GetGlossaryAsync<TKey>(TKey key)
|
||||
where TKey : notnull
|
||||
{
|
||||
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>("skglossary");
|
||||
return collection.GetAsync(key, new() { IncludeVectors = true });
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The type of the model key.</typeparam>
|
||||
/// <param name="uniqueKeyGenerator">A function that can be used to generate unique keys for the model in the type that the model requires.</param>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary<TKey>> CreateGlossaryEntries<TKey>(Func<TKey> uniqueKeyGenerator)
|
||||
{
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
/// <typeparam name="TKey">The type of the model key.</typeparam>
|
||||
private sealed class Glossary<TKey>
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public TKey Key { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text.Json;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Qdrant;
|
||||
using Qdrant.Client;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// A simple example showing how to ingest data into a vector store using <see cref="QdrantVectorStore"/>.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Create an embedding generator.
|
||||
/// 2. Create a Qdrant Vector Store.
|
||||
/// 3. Ingest some data into the vector store.
|
||||
/// 4. Read the data back from the vector store.
|
||||
///
|
||||
/// You need a local instance of Docker running, since the associated fixture will try and start a Qdrant container in the local docker instance to run against.
|
||||
/// </summary>
|
||||
[Collection("Sequential")]
|
||||
public class VectorStore_DataIngestion_Simple(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initiate the docker container and construct the vector store.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
|
||||
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateGlossaryEntries().ToList();
|
||||
var keys = glossaryEntries.Select(entry => entry.Key).ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection and return their keys.
|
||||
await collection.UpsertAsync(glossaryEntries);
|
||||
|
||||
// Retrieve one of the upserted records from the collection.
|
||||
var upsertedRecord = await collection.GetAsync(keys.First(), new() { IncludeVectors = true });
|
||||
|
||||
// Write upserted keys and one of the upserted records to the console.
|
||||
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class Glossary
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public ulong Key { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary> CreateGlossaryEntries()
|
||||
{
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 1,
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 2,
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 3,
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Text.Json;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.Qdrant;
|
||||
using Qdrant.Client;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// Semantic Kernel support dynamic data modeling for vector stores that can be used with any
|
||||
/// schema. The schema still has to be provided in the form of a record definition, but no
|
||||
/// custom .NET data model is required; a simple dictionary can be used.
|
||||
///
|
||||
/// The sample shows how to
|
||||
/// 1. Upsert data using dynamic data modeling and retrieve it from the vector store using a custom data model.
|
||||
/// 2. Upsert data using a custom data model and retrieve it from the vector store using the dynamic data modeling.
|
||||
/// </summary>
|
||||
public class VectorStore_DynamicDataModel_Interop(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
private static readonly JsonSerializerOptions s_indentedSerializerOptions = new() { WriteIndented = true };
|
||||
|
||||
private static readonly VectorStoreCollectionDefinition s_definition = new()
|
||||
{
|
||||
Properties =
|
||||
[
|
||||
new VectorStoreKeyProperty("Key", typeof(ulong)),
|
||||
new VectorStoreDataProperty("Term", typeof(string)),
|
||||
new VectorStoreDataProperty("Definition", typeof(string)),
|
||||
new VectorStoreVectorProperty("DefinitionEmbedding", typeof(ReadOnlyMemory<float>), 1536)
|
||||
]
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertWithDynamicRetrieveWithCustomAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initiate the docker container and construct the vector store.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
|
||||
|
||||
// Get and create collection if it doesn't exist using the dynamic data model and record definition that defines the schema.
|
||||
var dynamicDataModelCollection = vectorStore.GetDynamicCollection("skglossary", s_definition);
|
||||
await dynamicDataModelCollection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateDynamicGlossaryEntries().ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry["DefinitionEmbedding"] = (await embeddingGenerator.GenerateAsync((string)entry["Definition"]!)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection.
|
||||
await dynamicDataModelCollection.UpsertAsync(glossaryEntries);
|
||||
|
||||
// Get the collection using the custom data model.
|
||||
var customDataModelCollection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
|
||||
|
||||
// Retrieve one of the upserted records from the collection.
|
||||
var upsertedRecord = await customDataModelCollection.GetAsync((ulong)glossaryEntries.First()["Key"]!, new() { IncludeVectors = true });
|
||||
|
||||
// Write one of the upserted records to the console.
|
||||
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord, s_indentedSerializerOptions)}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertWithCustomRetrieveWithDynamicAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initiate the docker container and construct the vector store.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
var vectorStore = new QdrantVectorStore(new QdrantClient("localhost"), ownsClient: true);
|
||||
|
||||
// Get and create collection if it doesn't exist using the custom data model.
|
||||
var customDataModelCollection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
|
||||
await customDataModelCollection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateCustomGlossaryEntries().ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection and return their keys.
|
||||
await customDataModelCollection.UpsertAsync(glossaryEntries);
|
||||
|
||||
// Get the collection using the dynamic data model.
|
||||
var dynamicDataModelCollection = vectorStore.GetDynamicCollection("skglossary", s_definition);
|
||||
|
||||
// Retrieve one of the upserted records from the collection.
|
||||
var upsertedRecord = await dynamicDataModelCollection.GetAsync(glossaryEntries.First().Key, new() { IncludeVectors = true });
|
||||
|
||||
// Write one of the upserted records to the console.
|
||||
Console.WriteLine($"Upserted record: {JsonSerializer.Serialize(upsertedRecord, s_indentedSerializerOptions)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class Glossary
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public ulong Key { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries using the custom data model.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary> CreateCustomGlossaryEntries()
|
||||
{
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 1,
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data.",
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 2,
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc.",
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 3,
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt).",
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries using dynamic data modeling.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Dictionary<string, object?>> CreateDynamicGlossaryEntries()
|
||||
{
|
||||
yield return new Dictionary<string, object?>
|
||||
{
|
||||
["Key"] = 1ul,
|
||||
["Term"] = "API",
|
||||
["Definition"] = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Dictionary<string, object?>
|
||||
{
|
||||
["Key"] = 2ul,
|
||||
["Term"] = "Connectors",
|
||||
["Definition"] = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Dictionary<string, object?>
|
||||
{
|
||||
["Key"] = 3ul,
|
||||
["Term"] = "RAG",
|
||||
["Definition"] = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Azure.Search.Documents.Indexes;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// A simple example showing how to ingest data into a vector store and then use hybrid search to find related records to a given string and set of keywords.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Create an embedding generator.
|
||||
/// 2. Create an AzureAISearch Vector Store.
|
||||
/// 3. Ingest some data into the vector store.
|
||||
/// 4. Do a hybrid search on the vector store with various text+keyword and filtering options.
|
||||
/// </summary>
|
||||
public class VectorStore_HybridSearch_Simple_AzureAISearch(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task IngestDataAndUseHybridSearch()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct the AzureAISearch VectorStore.
|
||||
var searchIndexClient = new SearchIndexClient(
|
||||
new Uri(TestConfiguration.AzureAISearch.Endpoint),
|
||||
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
|
||||
var vectorStore = new AzureAISearchVectorStore(searchIndexClient);
|
||||
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<string, Glossary>("skglossary");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
var hybridSearchCollection = (IKeywordHybridSearchable<Glossary>)collection;
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateGlossaryEntries().ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection and return their keys.
|
||||
await collection.UpsertAsync(glossaryEntries);
|
||||
|
||||
// Search the collection using a vector search.
|
||||
var searchString = "What is an Application Programming Interface";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Application", "Programming", "Interface"], top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
Console.WriteLine();
|
||||
|
||||
// Search the collection using a vector search.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Retrieval", "Augmented", "Generation"], top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
Console.WriteLine();
|
||||
|
||||
// Search the collection using a vector search with pre-filtering.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await hybridSearchCollection.HybridSearchAsync(searchVector, ["Retrieval", "Augmented", "Generation"], top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Number of results: " + resultRecords.Count);
|
||||
Console.WriteLine("Result 1 Score: " + resultRecords[0].Score);
|
||||
Console.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
|
||||
Console.WriteLine("Result 2 Score: " + resultRecords[1].Score);
|
||||
Console.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class Glossary
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public string Key { get; set; }
|
||||
|
||||
[VectorStoreData(IsIndexed = true)]
|
||||
public string Category { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData(IsFullTextIndexed = true)]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary> CreateGlossaryEntries()
|
||||
{
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = "1",
|
||||
Category = "External Definitions",
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = "2",
|
||||
Category = "Core Definitions",
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = "3",
|
||||
Category = "External Definitions",
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreLangchainInterop;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Pinecone;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// Example showing how to consume data that had previously been ingested into a database using Langchain.
|
||||
/// The example also demonstrates how to get all vector stores to share the same data model, so where necessary
|
||||
/// a conversion is done, specifically for ids, where the database requires GUIDs, but we want to use strings
|
||||
/// containing GUIDs in the common data model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// To run these samples, you need to first create collections instances using Langhain.
|
||||
/// This sample assumes that you used the pets sample data set from this article:
|
||||
/// https://python.langchain.com/docs/tutorials/retrievers/#documents
|
||||
/// And the from_documents method to create the collection as shown here:
|
||||
/// https://python.langchain.com/docs/tutorials/retrievers/#vector-stores
|
||||
/// </remarks>
|
||||
public class VectorStore_Langchain_Interop(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
/// <summary>
|
||||
/// Shows how to read data from a Pinecone collection that was created and ingested using Langchain.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReadDataFromLangchainPineconeAsync()
|
||||
{
|
||||
var pineconeClient = new PineconeClient(TestConfiguration.Pinecone.ApiKey);
|
||||
var vectorStore = PineconeFactory.CreatePineconeLangchainInteropVectorStore(pineconeClient);
|
||||
await this.ReadDataFromCollectionAsync(vectorStore, "pets");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shows how to read data from a Redis collection that was created and ingested using Langchain.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReadDataFromLangchainRedisAsync()
|
||||
{
|
||||
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
|
||||
var vectorStore = RedisFactory.CreateRedisLangchainInteropVectorStore(database);
|
||||
await this.ReadDataFromCollectionAsync(vectorStore, "pets");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Method to do a vector search on a collection in the provided vector store.
|
||||
/// </summary>
|
||||
/// <param name="vectorStore">The vector store to search.</param>
|
||||
/// <param name="collectionName">The name of the collection.</param>
|
||||
/// <returns>An async task.</returns>
|
||||
private async Task ReadDataFromCollectionAsync(VectorStore vectorStore, string collectionName)
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator();
|
||||
|
||||
// Get the collection.
|
||||
var collection = vectorStore.GetCollection<string, LangchainDocument<string>>(collectionName);
|
||||
|
||||
// Search the data set.
|
||||
var searchString = "I'm looking for an animal that is loyal and will make a great companion";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
this.Output.WriteLine("Search string: " + searchString);
|
||||
this.Output.WriteLine("Source: " + resultRecords.First().Record.Source);
|
||||
this.Output.WriteLine("Text: " + resultRecords.First().Record.Content);
|
||||
this.Output.WriteLine();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Azure.Search.Documents.Indexes;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.AzureAISearch;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to use common code, that can work with any vector database, with an Azure AI Search instance.
|
||||
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
|
||||
/// The common code ingests data into the vector store and then searches over that data.
|
||||
/// This example is part of a set of examples each showing a different vector database.
|
||||
///
|
||||
/// For other databases, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
|
||||
///
|
||||
/// To run this sample, you need an already existing Azure AI Search instance.
|
||||
/// To set your secrets use:
|
||||
/// <para> dotnet user-secrets set "AzureAISearch:Endpoint" "https://... .search.windows.net"</para>
|
||||
/// <para> dotnet user-secrets set "AzureAISearch:ApiKey" "{Key from your Search service resource}"</para>
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiStore_AzureAISearch(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleWithDIAsync()
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
credential: new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Register the Azure AI Search VectorStore.
|
||||
kernelBuilder.Services.AddAzureAISearchVectorStore(
|
||||
new Uri(TestConfiguration.AzureAISearch.Endpoint),
|
||||
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
|
||||
|
||||
// Register the test output helper common processor with the DI container.
|
||||
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
|
||||
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a common processor object using the DI container.
|
||||
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Azure AI Search supports string, but others may not support strings.
|
||||
await processor.IngestDataAndSearchAsync("skglossary-with-di", () => Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExampleWithoutDIAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct the Azure AI Search VectorStore.
|
||||
var searchIndexClient = new SearchIndexClient(
|
||||
new Uri(TestConfiguration.AzureAISearch.Endpoint),
|
||||
new AzureKeyCredential(TestConfiguration.AzureAISearch.ApiKey));
|
||||
var vectorStore = new AzureAISearchVectorStore(searchIndexClient);
|
||||
|
||||
// Create the common processor that works for any vector store.
|
||||
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Azure AI Search supports string, but others may not support strings.
|
||||
await processor.IngestDataAndSearchAsync("skglossary-without-di", () => Guid.NewGuid().ToString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// This class is part of an example that shows how to ingest data into a vector store and then use vector search to find related records to a given string.
|
||||
/// The example shows how to write code that can be used with multiple database types.
|
||||
/// This class contains the common code.
|
||||
///
|
||||
/// For the entry point of the example for each database, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
|
||||
/// </summary>
|
||||
/// <param name="vectorStore">The vector store to ingest data into.</param>
|
||||
/// <param name="embeddingGenerator">The service to use for generating embeddings.</param>
|
||||
/// <param name="output">A helper to write output to the xUnit test output stream.</param>
|
||||
public class VectorStore_VectorSearch_MultiStore_Common(VectorStore vectorStore, IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, ITestOutputHelper output)
|
||||
{
|
||||
/// <summary>
|
||||
/// Ingest data into a collection with the given name, and search over that data.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The type of key to use for database records.</typeparam>
|
||||
/// <param name="collectionName">The name of the collection to ingest the data into.</param>
|
||||
/// <param name="uniqueKeyGenerator">A function to generate unique keys for each record to upsert.</param>
|
||||
/// <returns>An async task.</returns>
|
||||
public async Task IngestDataAndSearchAsync<TKey>(string collectionName, Func<TKey> uniqueKeyGenerator)
|
||||
where TKey : notnull
|
||||
{
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<TKey, Glossary<TKey>>(collectionName);
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateGlossaryEntries(uniqueKeyGenerator).ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection.
|
||||
await collection.UpsertAsync(glossaryEntries);
|
||||
|
||||
await Task.Delay(5000); // Add a wait to ensure that indexing completes before we continue.
|
||||
|
||||
// Search the collection using a vector search.
|
||||
var searchString = "What is an Application Programming Interface";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
output.WriteLine("Search string: " + searchString);
|
||||
output.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
output.WriteLine();
|
||||
|
||||
// Search the collection using a vector search.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
output.WriteLine("Search string: " + searchString);
|
||||
output.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
output.WriteLine();
|
||||
|
||||
// Search the collection using a vector search with pre-filtering.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await collection.SearchAsync(searchVector, top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
|
||||
|
||||
output.WriteLine("Search string: " + searchString);
|
||||
output.WriteLine("Number of results: " + resultRecords.Count);
|
||||
output.WriteLine("Result 1 Score: " + resultRecords[0].Score);
|
||||
output.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
|
||||
output.WriteLine("Result 2 Score: " + resultRecords[1].Score);
|
||||
output.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The type of the model key.</typeparam>
|
||||
/// <param name="uniqueKeyGenerator">A function that can be used to generate unique keys for the model in the type that the model requires.</param>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary<TKey>> CreateGlossaryEntries<TKey>(Func<TKey> uniqueKeyGenerator)
|
||||
{
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Category = "External Definitions",
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Category = "Core Definitions",
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Glossary<TKey>
|
||||
{
|
||||
Key = uniqueKeyGenerator(),
|
||||
Category = "External Definitions",
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
/// <typeparam name="TKey">The type of the model key.</typeparam>
|
||||
private sealed class Glossary<TKey>
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public TKey Key { get; set; }
|
||||
|
||||
[VectorStoreData(IsIndexed = true)]
|
||||
public string Category { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to use common code, that can work with any vector database, with the InMemory vector store.
|
||||
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
|
||||
/// The common code ingests data into the vector store and then searches over that data.
|
||||
/// This example is part of a set of examples each showing a different vector database.
|
||||
///
|
||||
/// For other databases, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiStore_InMemory(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleWithDIAsync()
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
credential: new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Register the InMemory VectorStore.
|
||||
kernelBuilder.Services.AddInMemoryVectorStore();
|
||||
|
||||
// Register the test output helper common processor with the DI container.
|
||||
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
|
||||
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a common processor object using the DI container.
|
||||
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. InMemory supports any comparable type, but others may only support string or Guid or ulong, etc.
|
||||
// For this example we'll use int.
|
||||
var uniqueId = 0;
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => uniqueId++);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExampleWithoutDIAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct the InMemory VectorStore.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
|
||||
// Create the common processor that works for any vector store.
|
||||
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. InMemory supports any comparable type, but others may only support string or Guid or ulong, etc.
|
||||
// For this example we'll use int.
|
||||
var uniqueId = 0;
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => uniqueId++);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.PgVector;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to use common code, that can work with any vector database, with a Postgres database.
|
||||
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
|
||||
/// The common code ingests data into the vector store and then searches over that data.
|
||||
/// This example is part of a set of examples each showing a different vector database.
|
||||
///
|
||||
/// For other databases, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
|
||||
///
|
||||
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Postgres container in the local docker instance.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiStore_Postgres(ITestOutputHelper output, VectorStorePostgresContainerFixture PostgresFixture) : BaseTest(output), IClassFixture<VectorStorePostgresContainerFixture>
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleWithDIAsync()
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
credential: new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Initialize the Postgres docker container via the fixtures and register the Postgres VectorStore.
|
||||
await PostgresFixture.ManualInitializeAsync();
|
||||
kernelBuilder.Services.AddPostgresVectorStore("Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;");
|
||||
|
||||
// Register the test output helper common processor with the DI container.
|
||||
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
|
||||
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a common processor object using the DI container.
|
||||
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Postgres supports Guid and ulong keys, but others may support strings only.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => Guid.NewGuid());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExampleWithoutDIAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initialize the Postgres docker container via the fixtures and construct the Postgres VectorStore.
|
||||
await PostgresFixture.ManualInitializeAsync();
|
||||
using PostgresVectorStore vectorStore = new("Host=localhost;Port=5432;Username=postgres;Password=example;Database=postgres;");
|
||||
|
||||
// Create the common processor that works for any vector store.
|
||||
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Postgres supports Guid and ulong keys, but others may support strings only.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => Guid.NewGuid());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.Qdrant;
|
||||
using Qdrant.Client;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to use common code, that can work with any vector database, with a Qdrant database.
|
||||
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
|
||||
/// The common code ingests data into the vector store and then searches over that data.
|
||||
/// This example is part of a set of examples each showing a different vector database.
|
||||
///
|
||||
/// For other databases, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
|
||||
///
|
||||
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Qdrant container in the local docker instance.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiStore_Qdrant(ITestOutputHelper output, VectorStoreQdrantContainerFixture qdrantFixture) : BaseTest(output), IClassFixture<VectorStoreQdrantContainerFixture>
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleWithDIAsync()
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
credential: new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Initialize the Qdrant docker container via the fixtures and register the Qdrant VectorStore.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
kernelBuilder.Services.AddQdrantVectorStore("localhost", https: false);
|
||||
|
||||
// Register the test output helper common processor with the DI container.
|
||||
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
|
||||
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a common processor object using the DI container.
|
||||
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Qdrant supports Guid and ulong keys, but others may support strings only.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => Guid.NewGuid());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExampleWithoutDIAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initialize the Qdrant docker container via the fixtures and construct the Qdrant VectorStore.
|
||||
await qdrantFixture.ManualInitializeAsync();
|
||||
var qdrantClient = new QdrantClient("localhost");
|
||||
var vectorStore = new QdrantVectorStore(qdrantClient, ownsClient: true);
|
||||
|
||||
// Create the common processor that works for any vector store.
|
||||
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Qdrant supports Guid and ulong keys, but others may support strings only.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => Guid.NewGuid());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Memory.VectorStoreFixtures;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.SemanticKernel;
|
||||
using Microsoft.SemanticKernel.Connectors.Redis;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to use common code, that can work with any vector database, with a Redis database.
|
||||
/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
|
||||
/// The common code ingests data into the vector store and then searches over that data.
|
||||
/// This example is part of a set of examples each showing a different vector database.
|
||||
///
|
||||
/// For other databases, see the following classes:
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
|
||||
/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
|
||||
///
|
||||
/// Redis supports two record storage types: Json and HashSet.
|
||||
/// Note the use of the <see cref="RedisStorageType"/> enum to specify the preferred storage type.
|
||||
///
|
||||
/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Redis container in the local docker instance.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiStore_Redis(ITestOutputHelper output, VectorStoreRedisContainerFixture redisFixture) : BaseTest(output), IClassFixture<VectorStoreRedisContainerFixture>
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(RedisStorageType.Json)]
|
||||
[InlineData(RedisStorageType.HashSet)]
|
||||
public async Task ExampleWithDIAsync(RedisStorageType redisStorageType)
|
||||
{
|
||||
// Use the kernel for DI purposes.
|
||||
var kernelBuilder = Kernel
|
||||
.CreateBuilder();
|
||||
|
||||
// Register an embedding generation service with the DI container.
|
||||
kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
|
||||
deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
|
||||
endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
|
||||
credential: new AzureCliCredential(),
|
||||
dimensions: 1536);
|
||||
|
||||
// Initialize the Redis docker container via the fixtures and register the Redis VectorStore with the preferred storage type.
|
||||
await redisFixture.ManualInitializeAsync();
|
||||
kernelBuilder.Services.AddRedisVectorStore("localhost:6379", new() { StorageType = redisStorageType });
|
||||
|
||||
// Register the test output helper common processor with the DI container.
|
||||
kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
|
||||
kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Build the kernel.
|
||||
var kernel = kernelBuilder.Build();
|
||||
|
||||
// Build a common processor object using the DI container.
|
||||
var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Redis supports string keys, but others may not support string.
|
||||
// Also note that we are appending the collection name with the storage type so that we have two separate collections,
|
||||
// since a redis index for JSON records cannot be used to index hashset documents, and vice versa.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithDI" + redisStorageType, () => Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(RedisStorageType.Json)]
|
||||
[InlineData(RedisStorageType.HashSet)]
|
||||
public async Task ExampleWithoutDIAsync(RedisStorageType redisStorageType)
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Initialize the Redis docker container via the fixtures and construct the Redis VectorStore with the preferred storage type.
|
||||
await redisFixture.ManualInitializeAsync();
|
||||
var database = ConnectionMultiplexer.Connect("localhost:6379").GetDatabase();
|
||||
var vectorStore = new RedisVectorStore(database, new() { StorageType = redisStorageType });
|
||||
|
||||
// Create the common processor that works for any vector store.
|
||||
var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
|
||||
|
||||
// Run the process and pass a key generator function to it, to generate unique record keys.
|
||||
// The key generator function is required, since different vector stores may require different key types.
|
||||
// E.g. Redis supports string keys, but others may not support string.
|
||||
// Also note that we are appending the collection name with the storage type so that we have two separate collections,
|
||||
// since a redis index for JSON records cannot be used to index hashset documents, and vice versa.
|
||||
await processor.IngestDataAndSearchAsync("skglossaryWithoutDI" + redisStorageType, () => Guid.NewGuid().ToString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to do vector search where there may be multiple vectors
|
||||
/// stored in each record and you want to specify which vector to search on.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Create an InMemory Vector Store.
|
||||
/// 2. Generate and add some test data entries.
|
||||
/// 3. Search for records based on a specified vector.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_MultiVector(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task VectorSearchWithMultiVectorRecordAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct an InMemory vector store.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<int, Product>("skproducts");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create product records and generate embeddings for them.
|
||||
var productRecords = CreateProductRecords().ToList();
|
||||
var tasks = productRecords.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
var descriptionEmbeddingTask = embeddingGenerator.GenerateAsync(entry.Description);
|
||||
var featureListEmbeddingTask = embeddingGenerator.GenerateAsync(string.Join("\n", entry.FeatureList));
|
||||
|
||||
entry.DescriptionEmbedding = (await descriptionEmbeddingTask).Vector;
|
||||
entry.FeatureListEmbedding = (await featureListEmbeddingTask).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the product records into the collection.
|
||||
await collection.UpsertAsync(productRecords);
|
||||
|
||||
// Search the store using the description embedding.
|
||||
var searchString = "I am looking for a reasonably priced coffee maker";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await collection.SearchAsync(
|
||||
searchVector, top: 1, new()
|
||||
{
|
||||
VectorProperty = r => r.DescriptionEmbedding
|
||||
}).ToListAsync();
|
||||
|
||||
WriteLine("Search string: " + searchString);
|
||||
WriteLine("Result: " + resultRecords.First().Record.Description);
|
||||
WriteLine("Score: " + resultRecords.First().Score);
|
||||
WriteLine();
|
||||
|
||||
// Search the store using the feature list embedding.
|
||||
searchString = "I am looking for a handheld vacuum cleaner that will remove pet hair";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await collection.SearchAsync(
|
||||
searchVector,
|
||||
top: 1,
|
||||
new()
|
||||
{
|
||||
VectorProperty = r => r.FeatureListEmbedding
|
||||
}).ToListAsync();
|
||||
|
||||
WriteLine("Search string: " + searchString);
|
||||
WriteLine("Result: " + resultRecords.First().Record.Description);
|
||||
WriteLine("Score: " + resultRecords.First().Score);
|
||||
WriteLine();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample product records.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample product records.</returns>
|
||||
private static IEnumerable<Product> CreateProductRecords()
|
||||
{
|
||||
yield return new Product
|
||||
{
|
||||
Key = 1,
|
||||
Description = "Premium coffee maker that allows you to make up to 20 types of drinks with one machine.",
|
||||
FeatureList = ["Milk Frother", "Easy to use", "One button operation", "Stylish design"]
|
||||
};
|
||||
|
||||
yield return new Product
|
||||
{
|
||||
Key = 2,
|
||||
Description = "Value coffee maker that gives you what you need at a good price.",
|
||||
FeatureList = ["Simple design", "Easy to clean"]
|
||||
};
|
||||
|
||||
yield return new Product
|
||||
{
|
||||
Key = 3,
|
||||
Description = "Efficient vacuum cleaner",
|
||||
FeatureList = ["1000W power", "Hard floor tool", "Bagless", "Corded"]
|
||||
};
|
||||
|
||||
yield return new Product
|
||||
{
|
||||
Key = 4,
|
||||
Description = "High performance handheld vacuum cleaner",
|
||||
FeatureList = ["Pet hair tool", "2000W power", "Hard floor tool", "Bagless", "Cordless"]
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that can store product information with a description and a feature list with embeddings for both.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class Product
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public int Key { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Description { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public List<string> FeatureList { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DescriptionEmbedding { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> FeatureListEmbedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// An example showing how to do paging when there are many records in the database and you want to page through these page by page.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Create an InMemory Vector Store.
|
||||
/// 2. Generate and add some test data entries.
|
||||
/// 3. Read the data back using vector search by paging through the results page by page.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_Paging(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task VectorSearchWithPagingAsync()
|
||||
{
|
||||
// Construct an InMemory vector store.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<int, TextSnippet>("skglossary");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create some test data entries.
|
||||
// We are not generating real embeddings here, just some random numbers
|
||||
// to keep the example simple.
|
||||
for (int i = 0; i < 1000; i++)
|
||||
{
|
||||
var text = $"This is a test text snippet {i}";
|
||||
var embedding = new ReadOnlyMemory<float>([i, i + 1, i + 2, i + 3]);
|
||||
var textSnippet = new TextSnippet { Key = i, Text = text, TextEmbedding = embedding };
|
||||
await collection.UpsertAsync(textSnippet);
|
||||
}
|
||||
|
||||
// Create a vector to search with.
|
||||
// We are not generating a real embedding here, just some random numbers
|
||||
// to keep the example simple.
|
||||
var searchVector = new ReadOnlyMemory<float>([0, 1, 2, 3]);
|
||||
|
||||
// Loop until there are no more results.
|
||||
var page = 0;
|
||||
var moreResults = true;
|
||||
while (moreResults)
|
||||
{
|
||||
// Get the next page of results by asking for 10 results, and using 'Skip' to skip the results from the previous pages.
|
||||
var currentPageResults = collection.SearchAsync(
|
||||
searchVector,
|
||||
top: 10,
|
||||
new()
|
||||
{
|
||||
Skip = page * 10
|
||||
});
|
||||
|
||||
// Print the results.
|
||||
var pageCount = 0;
|
||||
await foreach (var result in currentPageResults)
|
||||
{
|
||||
Console.WriteLine($"Key: {result.Record.Key}, Text: {result.Record.Text}");
|
||||
pageCount++;
|
||||
}
|
||||
|
||||
// Stop when we got back less than the requested number of results.
|
||||
moreResults = pageCount == 10;
|
||||
page++;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that can store some text and its embedding.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class TextSnippet
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public int Key { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Text { get; set; }
|
||||
|
||||
[VectorStoreVector(4)]
|
||||
public ReadOnlyMemory<float> TextEmbedding { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// A simple example showing how to ingest data into a vector store and then use vector search to find related records to a given string.
|
||||
///
|
||||
/// The example shows the following steps:
|
||||
/// 1. Create an embedding generator.
|
||||
/// 2. Create an InMemory Vector Store.
|
||||
/// 3. Ingest some data into the vector store.
|
||||
/// 4. Search the vector store with various text and filtering options.
|
||||
/// </summary>
|
||||
public class VectorStore_VectorSearch_Simple(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExampleAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
|
||||
.GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct an InMemory vector store.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
|
||||
// Get and create collection if it doesn't exist.
|
||||
var collection = vectorStore.GetCollection<ulong, Glossary>("skglossary");
|
||||
await collection.EnsureCollectionExistsAsync();
|
||||
|
||||
// Create glossary entries and generate embeddings for them.
|
||||
var glossaryEntries = CreateGlossaryEntries().ToList();
|
||||
var tasks = glossaryEntries.Select(entry => Task.Run(async () =>
|
||||
{
|
||||
entry.DefinitionEmbedding = (await embeddingGenerator.GenerateAsync(entry.Definition)).Vector;
|
||||
}));
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Upsert the glossary entries into the collection and return their keys.
|
||||
await collection.UpsertAsync(glossaryEntries);
|
||||
|
||||
// Search the collection using a vector search.
|
||||
var searchString = "What is an Application Programming Interface";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
Console.WriteLine();
|
||||
|
||||
// Search the collection using a vector search.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await collection.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Definition);
|
||||
Console.WriteLine();
|
||||
|
||||
// Search the collection using a vector search with pre-filtering.
|
||||
searchString = "What is Retrieval Augmented Generation";
|
||||
searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
resultRecords = await collection.SearchAsync(searchVector, top: 3, new() { Filter = g => g.Category == "External Definitions" }).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Number of results: " + resultRecords.Count);
|
||||
Console.WriteLine("Result 1 Score: " + resultRecords[0].Score);
|
||||
Console.WriteLine("Result 1: " + resultRecords[0].Record.Definition);
|
||||
Console.WriteLine("Result 2 Score: " + resultRecords[1].Score);
|
||||
Console.WriteLine("Result 2: " + resultRecords[1].Record.Definition);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a glossary entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class Glossary
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public ulong Key { get; set; }
|
||||
|
||||
[VectorStoreData(IsIndexed = true)]
|
||||
public string Category { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Term { get; set; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Definition { get; set; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> DefinitionEmbedding { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create some sample glossary entries.
|
||||
/// </summary>
|
||||
/// <returns>A list of sample glossary entries.</returns>
|
||||
private static IEnumerable<Glossary> CreateGlossaryEntries()
|
||||
{
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 1,
|
||||
Category = "External Definitions",
|
||||
Term = "API",
|
||||
Definition = "Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 2,
|
||||
Category = "Core Definitions",
|
||||
Term = "Connectors",
|
||||
Definition = "Connectors allow you to integrate with various services provide AI capabilities, including LLM, AudioToText, TextToAudio, Embedding generation, etc."
|
||||
};
|
||||
|
||||
yield return new Glossary
|
||||
{
|
||||
Key = 3,
|
||||
Category = "External Definitions",
|
||||
Term = "RAG",
|
||||
Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)."
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.ClientModel;
|
||||
using System.ClientModel.Primitives;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.VectorData;
|
||||
using Microsoft.SemanticKernel.Connectors.InMemory;
|
||||
using Microsoft.SemanticKernel.Data;
|
||||
using Resources;
|
||||
|
||||
namespace Memory;
|
||||
|
||||
/// <summary>
|
||||
/// Sample showing how to create an <see cref="InMemoryVectorStore"/> collection from a list of strings
|
||||
/// and then save it to disk so that it can be reloaded later.
|
||||
/// </summary>
|
||||
public class InMemoryVectorStore_LoadData(ITestOutputHelper output) : BaseTest(output)
|
||||
{
|
||||
[Fact]
|
||||
public async Task LoadStringListAndSearchAsync()
|
||||
{
|
||||
// Create a logging handler to output HTTP requests and responses
|
||||
var handler = new LoggingHandler(new HttpClientHandler(), this.Output);
|
||||
var httpClient = new HttpClient(handler);
|
||||
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new OpenAI.OpenAIClient(
|
||||
new ApiKeyCredential(TestConfiguration.OpenAI.ApiKey),
|
||||
new OpenAI.OpenAIClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) })
|
||||
.GetEmbeddingClient(TestConfiguration.OpenAI.EmbeddingModelId)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct an InMemory vector store.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
var collectionName = "records";
|
||||
|
||||
// Path to the file where the record collection will be saved to and loaded from.
|
||||
string filePath = Path.Combine(Path.GetTempPath(), "semantic-kernel-info.json");
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
// Read a list of text strings from a file, to load into a new record collection.
|
||||
var skInfo = EmbeddedResource.Read("semantic-kernel-info.txt");
|
||||
var lines = skInfo!.Split('\n');
|
||||
|
||||
// Delegate which will create a record.
|
||||
static DataModel CreateRecord(string text, ReadOnlyMemory<float> embedding)
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Key = Guid.NewGuid(),
|
||||
Text = text,
|
||||
Embedding = embedding
|
||||
};
|
||||
}
|
||||
|
||||
// Create a record collection from a list of strings using the provided delegate.
|
||||
var collection = await vectorStore.CreateCollectionFromListAsync<Guid, DataModel>(
|
||||
collectionName, lines, embeddingGenerator, CreateRecord);
|
||||
|
||||
// Save the record collection to a file stream.
|
||||
using (FileStream fileStream = new(filePath, FileMode.OpenOrCreate))
|
||||
{
|
||||
await vectorStore.SerializeCollectionAsJsonAsync<Guid, DataModel>(collectionName, fileStream);
|
||||
}
|
||||
}
|
||||
|
||||
// Load the record collection from the file stream and perform a search.
|
||||
using (FileStream fileStream = new(filePath, FileMode.Open))
|
||||
{
|
||||
var vectorSearch = await vectorStore.DeserializeCollectionFromJsonAsync<Guid, DataModel>(fileStream);
|
||||
|
||||
// Search the collection using a vector search.
|
||||
var searchString = "What is the Semantic Kernel?";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await vectorSearch!.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Text);
|
||||
Console.WriteLine();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadTextSearchResultsAndSearchAsync()
|
||||
{
|
||||
// Create an embedding generation service.
|
||||
var embeddingGenerator = new OpenAI.OpenAIClient(TestConfiguration.OpenAI.ApiKey)
|
||||
.GetEmbeddingClient(TestConfiguration.OpenAI.EmbeddingModelId)
|
||||
.AsIEmbeddingGenerator(1536);
|
||||
|
||||
// Construct an InMemory vector store.
|
||||
var vectorStore = new InMemoryVectorStore();
|
||||
var collectionName = "records";
|
||||
|
||||
// Read a list of text strings from a file, to load into a new record collection.
|
||||
var searchResultsJson = EmbeddedResource.Read("what-is-semantic-kernel.json");
|
||||
var searchResults = JsonSerializer.Deserialize<List<TextSearchResult>>(searchResultsJson!);
|
||||
|
||||
// Delegate which will create a record.
|
||||
static DataModel CreateRecord(TextSearchResult searchResult, ReadOnlyMemory<float> embedding)
|
||||
{
|
||||
return new()
|
||||
{
|
||||
Key = Guid.NewGuid(),
|
||||
Title = searchResult.Name,
|
||||
Text = searchResult.Value ?? string.Empty,
|
||||
Link = searchResult.Link,
|
||||
Embedding = embedding
|
||||
};
|
||||
}
|
||||
|
||||
// Create a record collection from a list of strings using the provided delegate.
|
||||
var vectorSearch = await vectorStore.CreateCollectionFromTextSearchResultsAsync<Guid, DataModel>(
|
||||
collectionName, searchResults!, embeddingGenerator, CreateRecord);
|
||||
|
||||
// Search the collection using a vector search.
|
||||
var searchString = "What is the Semantic Kernel?";
|
||||
var searchVector = (await embeddingGenerator.GenerateAsync(searchString)).Vector;
|
||||
var resultRecords = await vectorSearch!.SearchAsync(searchVector, top: 1).ToListAsync();
|
||||
|
||||
Console.WriteLine("Search string: " + searchString);
|
||||
Console.WriteLine("Result: " + resultRecords.First().Record.Text);
|
||||
Console.WriteLine();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sample model class that represents a record entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Note that each property is decorated with an attribute that specifies how the property should be treated by the vector store.
|
||||
/// This allows us to create a collection in the vector store and upsert and retrieve instances of this class without any further configuration.
|
||||
/// </remarks>
|
||||
private sealed class DataModel
|
||||
{
|
||||
[VectorStoreKey]
|
||||
public Guid Key { get; init; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string Text { get; init; }
|
||||
|
||||
[VectorStoreData]
|
||||
public string? Link { get; init; }
|
||||
|
||||
[VectorStoreVector(1536)]
|
||||
public ReadOnlyMemory<float> Embedding { get; init; }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user