chore: import upstream snapshot with attribution
CodeQL / Analyze (csharp) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:21:23 +08:00
commit b957a53def
5423 changed files with 863745 additions and 0 deletions
@@ -0,0 +1,115 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests;
public abstract class CollectionManagementTests<TKey>(VectorStoreFixture fixture) : IAsyncLifetime
where TKey : notnull
{
public Task InitializeAsync()
=> fixture.VectorStore.EnsureCollectionDeletedAsync(this.CollectionName);
[ConditionalFact]
public async Task Collection_Ensure_Exists_Delete()
{
var collection = this.GetCollection();
Assert.False(await collection.CollectionExistsAsync());
await collection.EnsureCollectionExistsAsync();
Assert.True(await collection.CollectionExistsAsync());
await collection.EnsureCollectionDeletedAsync();
Assert.False(await collection.CollectionExistsAsync());
// Deleting a non-existing collection does not throw
await fixture.TestStore.DefaultVectorStore.EnsureCollectionDeletedAsync(collection.Name);
}
[ConditionalFact]
public async Task EnsureCollectionExists_twice_does_not_throw()
{
var collection = this.GetCollection();
await collection.EnsureCollectionExistsAsync();
await collection.EnsureCollectionExistsAsync();
Assert.True(await collection.CollectionExistsAsync());
}
[ConditionalFact]
public async Task Store_CollectionExists()
{
var store = fixture.VectorStore;
var collection = this.GetCollection();
Assert.False(await store.CollectionExistsAsync(collection.Name));
await collection.EnsureCollectionExistsAsync();
Assert.True(await store.CollectionExistsAsync(collection.Name));
}
[ConditionalFact]
public async Task Store_DeleteCollection()
{
var store = fixture.VectorStore;
var collection = this.GetCollection();
await collection.EnsureCollectionExistsAsync();
await fixture.TestStore.DefaultVectorStore.EnsureCollectionDeletedAsync(collection.Name);
Assert.False(await collection.CollectionExistsAsync());
}
[ConditionalFact]
public async Task Store_ListCollections()
{
var store = fixture.VectorStore;
var collection = this.GetCollection();
Assert.Empty(await store.ListCollectionNamesAsync().Where(n => n == collection.Name).ToListAsync());
await collection.EnsureCollectionExistsAsync();
var name = Assert.Single(await store.ListCollectionNamesAsync().Where(n => n == collection.Name).ToListAsync());
Assert.Equal(collection.Name, name);
}
[ConditionalFact]
public void Collection_metadata()
{
var collection = this.GetCollection();
var collectionMetadata = (VectorStoreCollectionMetadata?)collection.GetService(typeof(VectorStoreCollectionMetadata));
Assert.NotNull(collectionMetadata);
Assert.NotNull(collectionMetadata.VectorStoreSystemName);
Assert.NotNull(collectionMetadata.CollectionName);
}
protected virtual string CollectionNameBase => nameof(CollectionManagementTests<object>);
public virtual string CollectionName => fixture.TestStore.AdjustCollectionName(this.CollectionNameBase);
public sealed class Record : TestRecord<TKey>
{
public string? Text { get; set; }
public int Number { get; set; }
public ReadOnlyMemory<float> Floats { get; set; }
}
public virtual VectorStoreCollection<TKey, Record> GetCollection()
=> fixture.TestStore.CreateCollection<TKey, Record>(this.CollectionName, this.CreateRecordDefinition());
public virtual VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)) { StorageName = "key" },
new VectorStoreDataProperty(nameof(Record.Text), typeof(string)) { StorageName = "text" },
new VectorStoreDataProperty(nameof(Record.Number), typeof(int)) { StorageName = "number" },
new VectorStoreVectorProperty(nameof(Record.Floats), typeof(ReadOnlyMemory<float>), 10) { IndexKind = fixture.TestStore.DefaultIndexKind }
]
};
public Task DisposeAsync() => Task.CompletedTask;
}
@@ -0,0 +1,283 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Linq.Expressions;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using Xunit;
namespace VectorData.ConformanceTests;
// The type argument object? might not be serializable, which may cause Test Explorer to not enumerate individual data rows. Consider using a type that is known to be serializable.
#pragma warning disable xUnit1045
public abstract class DependencyInjectionTests<TVectorStore, TCollection, TKey, TRecord> : DependencyInjectionTests<TKey>
where TVectorStore : VectorStore
where TCollection : VectorStoreCollection<TKey, TRecord>
where TKey : notnull
where TRecord : class
{
protected virtual string CollectionName => "collectionName";
protected abstract void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null);
public abstract IEnumerable<Func<IServiceCollection, object?, ServiceLifetime, IServiceCollection>> StoreDelegates { get; }
public abstract IEnumerable<Func<IServiceCollection, object?, string, ServiceLifetime, IServiceCollection>> CollectionDelegates { get; }
[Fact]
public void ServiceCollectionCantBeNull()
{
foreach (var registrationDelegate in this.StoreDelegates)
{
Assert.Throws<ArgumentNullException>(() => registrationDelegate(null!, null, ServiceLifetime.Singleton));
Assert.Throws<ArgumentNullException>(() => registrationDelegate(null!, "serviceKey", ServiceLifetime.Singleton));
}
foreach (var registrationDelegate in this.CollectionDelegates)
{
Assert.Throws<ArgumentNullException>(() => registrationDelegate(null!, null, this.CollectionName, ServiceLifetime.Singleton));
Assert.Throws<ArgumentNullException>(() => registrationDelegate(null!, "serviceKey", this.CollectionName, ServiceLifetime.Singleton));
}
}
[Fact]
public void CollectionNameCantBeNullOrEmpty()
{
const string EmptyCollectionName = "";
foreach (var registrationDelegate in this.CollectionDelegates)
{
IServiceCollection services = this.CreateServices();
Assert.Throws<ArgumentNullException>(() => registrationDelegate(services, null, null!, ServiceLifetime.Singleton));
Assert.Throws<ArgumentNullException>(() => registrationDelegate(services, "serviceKey", null!, ServiceLifetime.Singleton));
Assert.Throws<ArgumentException>(() => registrationDelegate(services, null, EmptyCollectionName, ServiceLifetime.Singleton));
Assert.Throws<ArgumentException>(() => registrationDelegate(services, "serviceKey", EmptyCollectionName, ServiceLifetime.Singleton));
}
}
[Theory, MemberData(nameof(LifetimesAndServiceKeys))]
public virtual void CanRegisterVectorStore(ServiceLifetime lifetime, object? serviceKey)
{
foreach (var registrationDelegate in this.StoreDelegates)
{
IServiceCollection services = this.CreateServices(serviceKey);
registrationDelegate(services, serviceKey, lifetime);
using ServiceProvider serviceProvider = services.BuildServiceProvider();
// let's ensure that concrete types are registered
Verify<TVectorStore>(serviceProvider, lifetime, serviceKey);
// and the abstraction too
Verify<VectorStore>(serviceProvider, lifetime, serviceKey);
}
}
[Theory, MemberData(nameof(LifetimesAndServiceKeys))]
public void CanRegisterCollections(ServiceLifetime lifetime, object? serviceKey)
{
foreach (var registrationDelegate in this.CollectionDelegates)
{
IServiceCollection services = this.CreateServices(serviceKey);
registrationDelegate(services, serviceKey, this.CollectionName, lifetime);
using ServiceProvider serviceProvider = services.BuildServiceProvider();
// Let's ensure that concrete types are registered.
Verify<TCollection>(serviceProvider, lifetime, serviceKey);
// And the VectorStoreCollection abstraction.
Verify<VectorStoreCollection<TKey, TRecord>>(serviceProvider, lifetime, serviceKey);
// And the IVectorSearchable abstraction.
Verify<IVectorSearchable<TRecord>>(serviceProvider, lifetime, serviceKey);
if (typeof(IKeywordHybridSearchable<TRecord>).IsAssignableFrom(typeof(TCollection)))
{
Verify<IKeywordHybridSearchable<TRecord>>(serviceProvider, lifetime, serviceKey);
}
else
{
Assert.Null(serviceProvider.GetService<IKeywordHybridSearchable<TRecord>>());
}
}
}
[Theory, MemberData(nameof(LifetimesAndServiceKeys))]
public virtual void CanRegisterConcreteTypeVectorStoreAfterSomeAbstractionHasBeenRegistered(ServiceLifetime lifetime, object? serviceKey)
{
foreach (var registrationDelegate in this.StoreDelegates)
{
IServiceCollection services = this.CreateServices(serviceKey);
// Users may be willing to register more than one IVectorStore implementation.
services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, (sp, key) => new FakeVectorStore(), lifetime));
registrationDelegate(services, serviceKey, lifetime);
using ServiceProvider serviceProvider = services.BuildServiceProvider();
// let's ensure that concrete types are registered
Verify<TVectorStore>(serviceProvider, lifetime, serviceKey);
}
}
[Theory, MemberData(nameof(LifetimesAndServiceKeys))]
public void CanRegisterConcreteTypeCollectionsAfterSomeAbstractionHasBeenRegistered(ServiceLifetime lifetime, object? serviceKey)
{
foreach (var registrationDelegate in this.CollectionDelegates)
{
IServiceCollection services = this.CreateServices(serviceKey);
// Users may be willing to register more than one VectorStoreCollection implementation.
services.Add(new ServiceDescriptor(typeof(VectorStoreCollection<TKey, TRecord>), serviceKey, (sp, key) => new FakeVectorStoreRecordCollection(), lifetime));
registrationDelegate(services, serviceKey, this.CollectionName, lifetime);
using ServiceProvider serviceProvider = services.BuildServiceProvider();
// let's ensure that concrete types are registered
Verify<TCollection>(serviceProvider, lifetime, serviceKey);
}
}
[Theory, MemberData(nameof(LifetimesAndServiceKeys))]
public void EmbeddingGeneratorIsResolved(ServiceLifetime lifetime, object? serviceKey)
{
foreach (var registrationDelegate in this.CollectionDelegates)
{
IServiceCollection services = this.CreateServices(serviceKey);
bool wasResolved = false;
services.AddSingleton<IEmbeddingGenerator>(sp =>
{
wasResolved = true;
return null!;
});
registrationDelegate(services, serviceKey, this.CollectionName, lifetime);
Assert.False(wasResolved); // it's lazy
using ServiceProvider serviceProvider = services.BuildServiceProvider();
using var collection = serviceKey is null
? serviceProvider.GetRequiredService<TCollection>()
: serviceProvider.GetRequiredKeyedService<TCollection>(serviceKey);
Assert.True(wasResolved);
}
}
#pragma warning disable CA1000 // Do not declare static members on generic types
public static TheoryData<ServiceLifetime, object?> LifetimesAndServiceKeys { get; } = GetLifetimesAndServiceKeys();
#pragma warning restore CA1000 // Do not declare static members on generic types
private static TheoryData<ServiceLifetime, object?> GetLifetimesAndServiceKeys()
{
TheoryData<ServiceLifetime, object?> result = [];
foreach (ServiceLifetime lifetime in new ServiceLifetime[] { ServiceLifetime.Scoped, ServiceLifetime.Singleton, ServiceLifetime.Transient })
{
result.Add(lifetime, null);
result.Add(lifetime, "key");
result.Add(lifetime, 8);
}
return result;
}
protected IServiceCollection CreateServices(object? serviceKey = null)
{
IServiceCollection services = new ServiceCollection();
#pragma warning disable CA2000 // Dispose objects before losing scope
ConfigurationManager configuration = new();
#pragma warning restore CA2000 // Dispose objects before losing scope
services.AddSingleton<IConfiguration>(configuration);
this.PopulateConfiguration(configuration, serviceKey);
return services;
}
private static void Verify<TService>(ServiceProvider serviceProvider, ServiceLifetime lifetime, object? serviceKey)
where TService : class
{
TService? serviceFromFirstScope, serviceFromSecondScope, secondServiceFromSecondScope;
using (IServiceScope scope1 = serviceProvider.CreateScope())
{
serviceFromFirstScope = Resolve<TService>(scope1.ServiceProvider, serviceKey);
}
using (IServiceScope scope2 = serviceProvider.CreateScope())
{
serviceFromSecondScope = Resolve<TService>(scope2.ServiceProvider, serviceKey);
secondServiceFromSecondScope = Resolve<TService>(scope2.ServiceProvider, serviceKey);
}
Assert.NotNull(serviceFromFirstScope);
Assert.NotNull(serviceFromSecondScope);
Assert.NotNull(secondServiceFromSecondScope);
switch (lifetime)
{
case ServiceLifetime.Singleton:
Assert.Same(serviceFromFirstScope, serviceFromSecondScope);
Assert.Same(serviceFromSecondScope, secondServiceFromSecondScope);
break;
case ServiceLifetime.Scoped:
Assert.NotSame(serviceFromFirstScope, serviceFromSecondScope);
Assert.Same(serviceFromSecondScope, secondServiceFromSecondScope);
break;
case ServiceLifetime.Transient:
Assert.NotSame(serviceFromFirstScope, serviceFromSecondScope);
Assert.NotSame(serviceFromSecondScope, secondServiceFromSecondScope);
break;
}
}
protected static string CreateConfigKey(string prefix, object? serviceKey, string suffix)
=> serviceKey is null ? $"{prefix}:{suffix}" : $"{prefix}:{serviceKey}:{suffix}";
private static TService Resolve<TService>(IServiceProvider serviceProvider, object? serviceKey = null) where TService : notnull
=> serviceKey is null
? serviceProvider.GetRequiredService<TService>()
: serviceProvider.GetRequiredKeyedService<TService>(serviceKey);
private sealed class FakeVectorStore : VectorStore
{
public override Task<bool> CollectionExistsAsync(string name, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override VectorStoreCollection<TKey1, TRecord1> GetCollection<TKey1, TRecord1>(string name, VectorStoreCollectionDefinition? definition = null) => throw new NotImplementedException();
public override VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(string name, VectorStoreCollectionDefinition? definition = null) => throw new NotImplementedException();
public override object? GetService(Type serviceType, object? serviceKey = null) => throw new NotImplementedException();
public override IAsyncEnumerable<string> ListCollectionNamesAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException();
}
private sealed class FakeVectorStoreRecordCollection : VectorStoreCollection<TKey, TRecord>
{
public override string Name => throw new NotImplementedException();
public override Task<bool> CollectionExistsAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task<TRecord?> GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord, bool>> filter, int top, FilteredRecordRetrievalOptions<TRecord>? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override object? GetService(Type serviceType, object? serviceKey = null) => throw new NotImplementedException();
public override IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<TInput>(TInput searchValue, int top, VectorSearchOptions<TRecord>? options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public override Task UpsertAsync(IEnumerable<TRecord> records, CancellationToken cancellationToken = default) => throw new NotImplementedException();
}
}
public abstract class DependencyInjectionTests<TKey>
{
public sealed class Record : TestRecord<TKey>
{
[VectorStoreData(StorageName = "number")]
public int Number { get; set; }
[VectorStoreVector(Dimensions: 3, StorageName = "embedding")]
public ReadOnlyMemory<float> Floats { get; set; }
}
}
@@ -0,0 +1,198 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests;
public abstract class DistanceFunctionTests<TKey>(DistanceFunctionTests<TKey>.Fixture fixture)
where TKey : notnull
{
[ConditionalFact]
public virtual Task CosineDistance()
=> this.Test(DistanceFunction.CosineDistance, 0, 2, 1, [0, 2, 1]);
[ConditionalFact]
public virtual Task CosineSimilarity()
=> this.Test(DistanceFunction.CosineSimilarity, 1, -1, 0, [0, 2, 1]);
[ConditionalFact]
public virtual Task DotProductSimilarity()
=> this.Test(DistanceFunction.DotProductSimilarity, 1, -1, 0, [0, 2, 1]);
[ConditionalFact]
public virtual Task NegativeDotProductSimilarity()
=> this.Test(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]);
[ConditionalFact]
public virtual Task EuclideanDistance()
=> this.Test(DistanceFunction.EuclideanDistance, 0, 2, 1.73, [0, 2, 1]);
[ConditionalFact]
public virtual Task EuclideanSquaredDistance()
=> this.Test(DistanceFunction.EuclideanSquaredDistance, 0, 4, 3, [0, 2, 1]);
[ConditionalFact]
public virtual Task HammingDistance()
=> this.Test(DistanceFunction.HammingDistance, 0, 1, 3, [0, 1, 2]);
[ConditionalFact]
public virtual Task ManhattanDistance()
=> this.Test(DistanceFunction.ManhattanDistance, 0, 2, 3, [0, 1, 2]);
protected virtual async Task Test(
string distanceFunction,
double expectedExactMatchScore,
double expectedOppositeScore,
double expectedOrthogonalScore,
int[] resultOrder)
{
using var collection = fixture.CreateCollection(distanceFunction);
await collection.EnsureCollectionDeletedAsync();
await collection.EnsureCollectionExistsAsync();
ReadOnlyMemory<float> baseVector = new([1, 0, 0, 0]);
ReadOnlyMemory<float> oppositeVector = new([-1, 0, 0, 0]);
ReadOnlyMemory<float> orthogonalVector = new([0f, -1f, -1f, 0f]);
double[] scoreDictionary =
[
expectedExactMatchScore,
expectedOppositeScore,
expectedOrthogonalScore
];
double[] expectedScores =
[
scoreDictionary[resultOrder[0]],
scoreDictionary[resultOrder[1]],
scoreDictionary[resultOrder[2]]
];
List<SearchRecord> insertedRecords =
[
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 1,
Vector = baseVector,
},
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 2,
Vector = oppositeVector,
},
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 3,
Vector = orthogonalVector,
}
];
SearchRecord[] expectedRecords =
[
insertedRecords[resultOrder[0]],
insertedRecords[resultOrder[1]],
insertedRecords[resultOrder[2]]
];
await collection.UpsertAsync(insertedRecords);
await fixture.TestStore.WaitForDataAsync(collection, insertedRecords.Count, vectorSize: 4);
var results = await collection.SearchAsync(baseVector, top: 3).ToListAsync();
Assert.Equal(expectedRecords.Length, results.Count);
for (int i = 0; i < results.Count; i++)
{
Assert.Equal(expectedRecords[i].Key, results[i].Record.Key);
Assert.Equal(expectedRecords[i].Int, results[i].Record.Int);
if (fixture.AssertScores)
{
Assert.Equal(Math.Round(expectedScores[i], 2), Math.Round(results[i].Score!.Value, 2));
}
}
await this.TestScoreThreshold(collection);
}
protected virtual async Task TestScoreThreshold(VectorStoreCollection<TKey, SearchRecord> collection)
{
if (!fixture.TestStore.SupportsScoreThreshold)
{
await Assert.ThrowsAsync<NotSupportedException>(async () =>
{
await collection
.SearchAsync(
new ReadOnlyMemory<float>([1, 0, 0, 0]),
top: 3,
new() { ScoreThreshold = 0.9 })
.ToListAsync();
});
return;
}
// Fetch the top three records, then use the second's returned score as the threshold.
var results = await collection
.SearchAsync(new ReadOnlyMemory<float>([1, 0, 0, 0]), top: 3)
.ToListAsync();
var threshold = results[1].Score;
var filteredResults = await collection
.SearchAsync(
new ReadOnlyMemory<float>([1, 0, 0, 0]),
top: 3,
new() { ScoreThreshold = threshold })
.ToListAsync();
// Some providers use inclusive thresholds (>=), returning 2 results (first and second),
// while others use exclusive thresholds (>), returning only 1 result (first).
Assert.True(filteredResults.Count is 1 or 2);
Assert.Equal(results[0].Record.Key, filteredResults[0].Record.Key);
if (filteredResults.Count == 2)
{
Assert.Equal(results[1].Record.Key, filteredResults[1].Record.Key);
}
}
public abstract class Fixture : VectorStoreFixture
{
protected virtual string CollectionNameBase => nameof(DistanceFunctionTests<int>);
public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
protected virtual string? IndexKind => null;
public virtual bool AssertScores { get; } = true;
public virtual VectorStoreCollection<TKey, SearchRecord> CreateCollection(string distanceFunction)
{
VectorStoreCollectionDefinition definition = new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(SearchRecord.Key), typeof(TKey)),
new VectorStoreDataProperty(nameof(SearchRecord.Int), typeof(int)),
new VectorStoreVectorProperty(nameof(SearchRecord.Vector), typeof(ReadOnlyMemory<float>), dimensions: 4)
{
DistanceFunction = distanceFunction,
IndexKind = this.IndexKind ?? this.DefaultIndexKind
}
]
};
return this.TestStore.CreateCollection<TKey, SearchRecord>(this.CollectionName, definition);
}
}
public class SearchRecord
{
public TKey Key { get; set; } = default!;
public int Int { get; set; }
public ReadOnlyMemory<float> Vector { get; set; }
}
}
@@ -0,0 +1,623 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests;
#pragma warning disable CA2000 // Don't actually need to dispose FakeEmbeddingGenerator
#pragma warning disable CS8605 // Unboxing a possibly null value.
public abstract class EmbeddingGenerationTests<TKey>(EmbeddingGenerationTests<TKey>.StringVectorFixture stringVectorFixture, EmbeddingGenerationTests<TKey>.RomOfFloatVectorFixture romOfFloatVectorFixture)
where TKey : notnull
{
#region Search
[ConditionalFact]
public virtual async Task SearchAsync_with_property_generator()
{
// Property level: embedding generators are defined at all levels. The property generator should take precedence.
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Property ([1, 1, 3])", result.Record.Text);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_property_generator_dynamic()
{
// Property level: embedding generators are defined at all levels. The property generator should take precedence.
var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Property ([1, 1, 3])", result.Record[nameof(Record.Text)]);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_collection_generator()
{
// Collection level: embedding generators are defined at the collection and store level - the collection generator should take precedence.
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: false);
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Collection ([1, 1, 2])", result.Record.Text);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_store_generator()
{
// Store level: an embedding generator is defined at the store level only.
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: false, propertyGenerator: false);
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Store ([1, 1, 1])", result.Record.Text);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_store_dependency_injection()
{
foreach (var registrationDelegate in stringVectorFixture.DependencyInjectionStoreRegistrationDelegates)
{
IServiceCollection serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<IEmbeddingGenerator>(new FakeEmbeddingGenerator(replaceLast: 1));
registrationDelegate(serviceCollection);
await using var serviceProvider = serviceCollection.BuildServiceProvider();
var vectorStore = serviceProvider.GetRequiredService<VectorStore>();
var collection = vectorStore.GetCollection<TKey, Record>(stringVectorFixture.CollectionName, stringVectorFixture.CreateRecordDefinition());
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Store ([1, 1, 1])", result.Record.Text);
}
}
[ConditionalFact]
public virtual async Task SearchAsync_with_collection_dependency_injection()
{
foreach (var registrationDelegate in stringVectorFixture.DependencyInjectionCollectionRegistrationDelegates)
{
IServiceCollection serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<IEmbeddingGenerator>(new FakeEmbeddingGenerator(replaceLast: 1));
registrationDelegate(serviceCollection);
await using var serviceProvider = serviceCollection.BuildServiceProvider();
var collection = serviceProvider.GetRequiredService<VectorStoreCollection<TKey, RecordWithAttributes>>();
var result = await collection.SearchAsync("[1, 1, 0]", top: 1).SingleAsync();
Assert.Equal("Store ([1, 1, 1])", result.Record.Text);
}
}
[ConditionalFact]
public virtual async Task SearchAsync_with_custom_input_type()
{
var recordDefinition = new VectorStoreCollectionDefinition()
{
Properties = stringVectorFixture.CreateRecordDefinition().Properties
.Select(p => p is VectorStoreVectorProperty vectorProperty
? new VectorStoreVectorProperty<Customer>(nameof(Record.Embedding), dimensions: 3)
{
DistanceFunction = stringVectorFixture.DefaultDistanceFunction,
IndexKind = stringVectorFixture.DefaultIndexKind
}
: p)
.ToList()
};
var collection = stringVectorFixture.GetCollection<RecordWithCustomerVectorProperty>(
stringVectorFixture.CreateVectorStore(new FakeCustomerEmbeddingGenerator([1, 1, 1])),
stringVectorFixture.CollectionName,
recordDefinition);
var result = await collection.SearchAsync(new Customer(), top: 1).SingleAsync();
Assert.Equal("Store ([1, 1, 1])", result.Record.Text);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_search_only_embedding_generation()
{
var collection = romOfFloatVectorFixture.GetCollection<RecordWithRomOfFloatVectorProperty>(
romOfFloatVectorFixture.CreateVectorStore(new FakeEmbeddingGenerator()),
romOfFloatVectorFixture.CollectionName,
romOfFloatVectorFixture.CreateRecordDefinition());
var result = await collection.SearchAsync("[1, 1, 1]", top: 1).SingleAsync();
Assert.Equal("Store ([1, 1, 1])", result.Record.Text);
}
[ConditionalFact]
public virtual async Task SearchAsync_string_without_generator_throws()
{
// The database doesn't support embedding generation, and no client-side generator has been configured at any level,
// so SearchAsync should throw for string.
var collection = stringVectorFixture.GetCollection<RawRecord>(stringVectorFixture.TestStore.DefaultVectorStore, stringVectorFixture.CollectionName + "withoutgenerator");
var exception = await Assert.ThrowsAsync<NotSupportedException>(() => collection.SearchAsync("foo", top: 1).ToListAsync().AsTask());
Assert.StartsWith(
"A value of type 'string' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are:",
exception.Message);
}
public class RawRecord
{
[VectorStoreKey]
public TKey Key { get; set; } = default!;
[VectorStoreVector(Dimensions: 3)]
public ReadOnlyMemory<float> Embedding { get; set; }
}
[ConditionalFact]
public virtual async Task SearchAsync_with_incompatible_generator_throws()
{
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
// We have a generator configured for string, not int.
var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => collection.SearchAsync(8, top: 1).ToListAsync().AsTask());
Assert.Equal($"An input of type 'int' was provided, but an incompatible embedding generator of type '{nameof(FakeEmbeddingGenerator)}' was configured.", exception.Message);
}
#endregion Search
#region Upsert
[ConditionalFact]
public virtual async Task UpsertAsync()
{
var counter = stringVectorFixture.GenerateNextCounter();
var record = new Record
{
Key = stringVectorFixture.GenerateNextKey<TKey>(),
Embedding = "[100, 1, 0]",
Counter = counter,
Text = nameof(UpsertAsync)
};
// Property level: embedding generators are defined at all levels. The property generator should take precedence.
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
await collection.UpsertAsync(record).ConfigureAwait(false);
await stringVectorFixture.TestStore.WaitForDataAsync(collection, 1, filter: r => r.Counter == counter);
var result = await collection.SearchAsync(new ReadOnlyMemory<float>([100, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter, result.Record.Counter);
}
[ConditionalFact]
public virtual async Task UpsertAsync_dynamic()
{
var counter = stringVectorFixture.GenerateNextCounter();
var record = new Dictionary<string, object?>
{
[nameof(Record.Key)] = stringVectorFixture.GenerateNextKey<TKey>(),
[nameof(Record.Embedding)] = "[200, 1, 0]",
[nameof(Record.Counter)] = counter,
[nameof(Record.Text)] = nameof(UpsertAsync_dynamic)
};
// Property level: embedding generators are defined at all levels. The property generator should take precedence.
var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
await collection.UpsertAsync(record).ConfigureAwait(false);
await stringVectorFixture.TestStore.WaitForDataAsync(collection, 1, filter: r => (int)r[nameof(Record.Counter)] == counter);
var result = await collection.SearchAsync(new ReadOnlyMemory<float>([200, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter, result.Record[nameof(Record.Counter)]);
}
[ConditionalFact]
public virtual async Task UpsertAsync_batch()
{
var (counter1, counter2) = (stringVectorFixture.GenerateNextCounter(), stringVectorFixture.GenerateNextCounter());
Record[] records =
[
new()
{
Key = stringVectorFixture.GenerateNextKey<TKey>(),
Embedding = "[300, 1, 0]",
Counter = counter1,
Text = nameof(UpsertAsync_batch) + "1"
},
new()
{
Key = stringVectorFixture.GenerateNextKey<TKey>(),
Embedding = "[400, 1, 0]",
Counter = counter2,
Text = nameof(UpsertAsync_batch) + "2"
}
];
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
await collection.UpsertAsync(records).ConfigureAwait(false);
await stringVectorFixture.TestStore.WaitForDataAsync(collection, 2, filter: r => (int)r.Counter == counter1 || (int)r.Counter == counter2);
var result = await collection.SearchAsync(new ReadOnlyMemory<float>([300, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter1, result.Record.Counter);
result = await collection.SearchAsync(new ReadOnlyMemory<float>([400, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter2, result.Record.Counter);
}
[ConditionalFact]
public virtual async Task UpsertAsync_batch_dynamic()
{
var (counter1, counter2) = (stringVectorFixture.GenerateNextCounter(), stringVectorFixture.GenerateNextCounter());
Dictionary<string, object?>[] records =
[
new()
{
[nameof(Record.Key)] = stringVectorFixture.GenerateNextKey<TKey>(),
[nameof(Record.Embedding)] = "[500, 1, 0]",
[nameof(Record.Counter)] = counter1,
[nameof(Record.Text)] = nameof(UpsertAsync_batch_dynamic) + "1"
},
new()
{
[nameof(Record.Key)] = stringVectorFixture.GenerateNextKey<TKey>(),
[nameof(Record.Embedding)] = "[600, 1, 0]",
[nameof(Record.Counter)] = counter2,
[nameof(Record.Text)] = nameof(UpsertAsync_batch_dynamic) + "2"
}
];
var collection = this.GetDynamicCollection(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
await collection.UpsertAsync(records).ConfigureAwait(false);
await stringVectorFixture.TestStore.WaitForDataAsync(collection, 2, filter: r => (int)r[nameof(Record.Counter)] == counter1 || (int)r[nameof(Record.Counter)] == counter2);
var result = await collection.SearchAsync(new ReadOnlyMemory<float>([500, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter1, result.Record[nameof(Record.Counter)]);
result = await collection.SearchAsync(new ReadOnlyMemory<float>([600, 1, 3]), top: 1).SingleAsync();
Assert.Equal(counter2, result.Record[nameof(Record.Counter)]);
}
#endregion Upsert
#region IncludeVectors
[ConditionalFact]
public virtual async Task SearchAsync_with_IncludeVectors_throws()
{
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
var exception = await Assert.ThrowsAsync<NotSupportedException>(() => collection.SearchAsync("[1, 0, 0]", top: 1, new() { IncludeVectors = true }).ToListAsync().AsTask());
Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message);
}
[ConditionalFact]
public virtual async Task GetAsync_with_IncludeVectors_throws()
{
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
var exception = await Assert.ThrowsAsync<NotSupportedException>(() => collection.GetAsync(stringVectorFixture.TestData[0].Key, new() { IncludeVectors = true }));
Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message);
}
[ConditionalFact]
public virtual async Task GetAsync_enumerable_with_IncludeVectors_throws()
{
var collection = this.GetCollection<Record>(storeGenerator: true, collectionGenerator: true, propertyGenerator: true);
var exception = await Assert.ThrowsAsync<NotSupportedException>(() =>
collection.GetAsync(
[stringVectorFixture.TestData[0].Key, stringVectorFixture.TestData[1].Key],
new() { IncludeVectors = true })
.ToListAsync().AsTask());
Assert.Equal("When an embedding generator is configured, `Include Vectors` cannot be enabled.", exception.Message);
}
#endregion IncludeVectors
#region Support
public class Record : TestRecord<TKey>
{
public string? Embedding { get; set; }
public int Counter { get; set; }
public string? Text { get; set; }
}
public class RecordWithAttributes
{
[VectorStoreKey]
public TKey Key { get; set; } = default!;
[VectorStoreVector(Dimensions: 3)]
public string? Embedding { get; set; }
[VectorStoreData(IsIndexed = true)]
public int Counter { get; set; }
[VectorStoreData]
public string? Text { get; set; }
}
public class RecordWithCustomerVectorProperty
{
public TKey Key { get; set; } = default!;
public Customer? Embedding { get; set; }
public int Counter { get; set; }
public string? Text { get; set; }
}
public class RecordWithRomOfFloatVectorProperty : TestRecord<TKey>
{
public ReadOnlyMemory<float> Embedding { get; set; }
public int Counter { get; set; }
public string? Text { get; set; }
}
public class Customer
{
public string? FirstName { get; set; }
public string? LastName { get; set; }
}
private VectorStoreCollection<TKey, TRecord> GetCollection<TRecord>(
bool storeGenerator = false,
bool collectionGenerator = false,
bool propertyGenerator = false)
where TRecord : class
{
var (vectorStore, recordDefinition) = this.GetStoreAndRecordDefinition(storeGenerator, collectionGenerator, propertyGenerator);
return stringVectorFixture.GetCollection<TRecord>(vectorStore, stringVectorFixture.CollectionName, recordDefinition);
}
private VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(
bool storeGenerator = false,
bool collectionGenerator = false,
bool propertyGenerator = false)
{
var (vectorStore, recordDefinition) = this.GetStoreAndRecordDefinition(storeGenerator, collectionGenerator, propertyGenerator);
return stringVectorFixture.GetDynamicCollection(vectorStore, stringVectorFixture.CollectionName, recordDefinition);
}
private (VectorStore, VectorStoreCollectionDefinition) GetStoreAndRecordDefinition(
bool storeGenerator = false,
bool collectionGenerator = false,
bool propertyGenerator = false)
{
var recordDefinition = stringVectorFixture.CreateRecordDefinition();
if (propertyGenerator)
{
foreach (var vectorProperty in recordDefinition.Properties.OfType<VectorStoreVectorProperty>())
{
vectorProperty.EmbeddingGenerator = new FakeEmbeddingGenerator(replaceLast: 3);
}
}
if (collectionGenerator)
{
recordDefinition.EmbeddingGenerator = new FakeEmbeddingGenerator(replaceLast: 2);
}
var vectorStore = stringVectorFixture.CreateVectorStore(storeGenerator ? new FakeEmbeddingGenerator(replaceLast: 1) : null);
return (vectorStore, recordDefinition);
}
public abstract class StringVectorFixture : VectorStoreCollectionFixture<TKey, Record>
{
private int _counter;
protected override string CollectionNameBase => nameof(EmbeddingGenerationTests<int>);
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(Record.Embedding), typeof(string), dimensions: 3)
{
DistanceFunction = this.DefaultDistanceFunction,
IndexKind = this.DefaultIndexKind
},
new VectorStoreDataProperty(nameof(Record.Counter), typeof(int)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(Record.Text), typeof(string))
],
EmbeddingGenerator = new FakeEmbeddingGenerator()
};
protected override List<Record> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = "[1, 1, 1]",
Counter = this.GenerateNextCounter(),
Text = "Store ([1, 1, 1])"
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = "[1, 1, 2]",
Counter = this.GenerateNextCounter(),
Text = "Collection ([1, 1, 2])"
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = "[1, 1, 3]",
Counter = this.GenerateNextCounter(),
Text = "Property ([1, 1, 3])"
}
];
public virtual VectorStoreCollection<TKey, TRecord> GetCollection<TRecord>(
VectorStore vectorStore,
string collectionName,
VectorStoreCollectionDefinition? recordDefinition = null)
where TRecord : class
=> vectorStore.GetCollection<TKey, TRecord>(collectionName, recordDefinition);
public virtual VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(
VectorStore vectorStore,
string collectionName,
VectorStoreCollectionDefinition recordDefinition)
=> vectorStore.GetDynamicCollection(collectionName, recordDefinition);
public abstract VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null);
public abstract Func<IServiceCollection, IServiceCollection>[] DependencyInjectionStoreRegistrationDelegates { get; }
public abstract Func<IServiceCollection, IServiceCollection>[] DependencyInjectionCollectionRegistrationDelegates { get; }
public virtual int GenerateNextCounter()
=> Interlocked.Increment(ref this._counter);
}
public abstract class RomOfFloatVectorFixture : VectorStoreCollectionFixture<TKey, RecordWithRomOfFloatVectorProperty>
{
private int _counter;
protected override string CollectionNameBase => "SearchOnlyEmbeddingGenerationTests";
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordWithRomOfFloatVectorProperty.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(RecordWithRomOfFloatVectorProperty.Embedding), typeof(ReadOnlyMemory<float>), dimensions: 3)
{
DistanceFunction = this.DefaultDistanceFunction,
IndexKind = this.DefaultIndexKind
},
new VectorStoreDataProperty(nameof(RecordWithRomOfFloatVectorProperty.Counter), typeof(int)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(RecordWithRomOfFloatVectorProperty.Text), typeof(string))
],
EmbeddingGenerator = new FakeEmbeddingGenerator()
};
protected override List<RecordWithRomOfFloatVectorProperty> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = new ReadOnlyMemory<float>([1, 1, 1]),
Counter = this.GenerateNextCounter(),
Text = "Store ([1, 1, 1])",
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = new ReadOnlyMemory<float>([1, 1, 2]),
Counter = this.GenerateNextCounter(),
Text = "Collection ([1, 1, 2])"
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Embedding = new ReadOnlyMemory<float>([1, 1, 3]),
Counter = this.GenerateNextCounter(),
Text = "Property ([1, 1, 3])"
}
];
public virtual VectorStoreCollection<TKey, TRecord> GetCollection<TRecord>(
VectorStore vectorStore,
string collectionName,
VectorStoreCollectionDefinition? recordDefinition = null)
where TRecord : class
=> vectorStore.GetCollection<TKey, TRecord>(collectionName, recordDefinition);
public virtual VectorStoreCollection<object, Dictionary<string, object?>> GetDynamicCollection(
VectorStore vectorStore,
string collectionName,
VectorStoreCollectionDefinition recordDefinition)
=> vectorStore.GetDynamicCollection(collectionName, recordDefinition);
public abstract VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator = null);
public abstract Func<IServiceCollection, IServiceCollection>[] DependencyInjectionStoreRegistrationDelegates { get; }
public abstract Func<IServiceCollection, IServiceCollection>[] DependencyInjectionCollectionRegistrationDelegates { get; }
public virtual int GenerateNextCounter()
=> Interlocked.Increment(ref this._counter);
}
protected sealed class FakeEmbeddingGenerator(int? replaceLast = null) : IEmbeddingGenerator<string, Embedding<float>>
{
public Task<GeneratedEmbeddings<Embedding<float>>> GenerateAsync(
IEnumerable<string> values,
EmbeddingGenerationOptions? options = null,
CancellationToken cancellationToken = default)
{
var results = new GeneratedEmbeddings<Embedding<float>>();
foreach (var value in values)
{
var vector = value.TrimStart('[').TrimEnd(']').Split(',').Select(s => float.Parse(s.Trim())).ToArray();
if (replaceLast is not null)
{
vector[vector.Length - 1] = replaceLast.Value;
}
results.Add(new Embedding<float>(vector));
}
return Task.FromResult(results);
}
public object? GetService(Type serviceType, object? serviceKey = null)
=> null;
public void Dispose()
{
}
}
private sealed class FakeCustomerEmbeddingGenerator(float[] embedding) : IEmbeddingGenerator<Customer, Embedding<float>>
{
public Task<GeneratedEmbeddings<Embedding<float>>> GenerateAsync(IEnumerable<Customer> values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default)
=> Task.FromResult(new GeneratedEmbeddings<Embedding<float>> { new(embedding) });
public object? GetService(Type serviceType, object? serviceKey = null)
=> null;
public void Dispose()
{
}
}
#endregion Support
}
@@ -0,0 +1,722 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Linq.Expressions;
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
#pragma warning disable CS8605 // Unboxing a possibly null value.
#pragma warning disable CS0252 // Possible unintended reference comparison; left hand side needs cast
#pragma warning disable RCS1098 // Constant values should be placed on right side of comparisons
namespace VectorData.ConformanceTests;
public abstract class FilterTests<TKey>(FilterTests<TKey>.Fixture fixture)
where TKey : notnull
{
#region Equality
[ConditionalFact]
public virtual Task Equal_with_int()
=> this.TestFilterAsync(
r => r.Int == 8,
r => (int)r["Int"] == 8);
[ConditionalFact]
public virtual Task Equal_with_string()
=> this.TestFilterAsync(
r => r.String == "foo",
r => r["String"] == "foo");
[ConditionalFact]
public virtual Task Equal_with_string_sql_injection_in_value()
{
string sqlInjection = $"foo; DROP TABLE {fixture.Collection.Name};";
return this.TestFilterAsync(
r => r.String == sqlInjection,
r => r["String"] == sqlInjection,
expectZeroResults: true);
}
[ConditionalFact]
public virtual async Task Equal_with_string_sql_injection_in_name()
{
if (fixture.TestDynamic)
{
await Assert.ThrowsAsync<InvalidOperationException>(
async () => await fixture.DynamicCollection.SearchAsync(
new ReadOnlyMemory<float>([1, 2, 3]),
top: 1,
new() { Filter = r => r["String = \"not\"; DROP TABLE FilterTests;"] == "" }).ToListAsync());
}
}
[ConditionalFact]
public virtual Task Equal_with_string_containing_special_characters()
=> this.TestFilterAsync(
r => r.String == fixture.SpecialCharactersText,
r => r["String"] == fixture.SpecialCharactersText);
[ConditionalFact]
public virtual Task Equal_with_string_is_not_Contains()
=> this.TestFilterAsync(
r => r.String == "some",
r => r["String"] == "some",
expectZeroResults: true);
[ConditionalFact]
public virtual Task Equal_reversed()
=> this.TestFilterAsync(
r => 8 == r.Int,
r => 8 == (int)r["Int"]);
[ConditionalFact]
public virtual Task Equal_with_null_reference_type()
=> this.TestFilterAsync(
r => r.String == null,
r => r["String"] == null);
[ConditionalFact]
public virtual Task Equal_with_null_captured()
{
string? s = null;
return this.TestFilterAsync(
r => r.String == s,
r => r["String"] == s);
}
[ConditionalFact]
public virtual Task Equal_int_property_with_nonnull_nullable_int()
{
int? i = 8;
return this.TestFilterAsync(
r => r.Int == i,
r => (int)r["Int"] == i);
}
[ConditionalFact]
public virtual Task Equal_int_property_with_null_nullable_int()
{
int? i = null;
return this.TestFilterAsync(
r => r.Int == i,
r => (int)r["Int"] == i,
expectZeroResults: true);
}
[ConditionalFact]
public virtual Task Equal_int_property_with_nonnull_nullable_int_Value()
{
int? i = 8;
return this.TestFilterAsync(
r => r.Int == i.Value,
r => (int)r["Int"] == i.Value);
}
#pragma warning disable CS8629 // Nullable value type may be null.
[ConditionalFact]
public virtual async Task Equal_int_property_with_null_nullable_int_Value()
{
int? i = null;
// TODO: Some connectors wrap filter translation exceptions in a VectorStoreException (#11766)
var exception = await Assert.ThrowsAnyAsync<Exception>(() => this.TestFilterAsync(
r => r.Int == i.Value,
r => (int)r["Int"] == i.Value,
expectZeroResults: true));
if (exception is not InvalidOperationException and not VectorStoreException { InnerException: InvalidOperationException })
{
Assert.Fail($"Expected {nameof(InvalidOperationException)} or {nameof(VectorStoreException)} but got {exception.GetType()}");
}
}
#pragma warning restore CS8629
[ConditionalFact]
public virtual Task NotEqual_with_int()
=> this.TestFilterAsync(
r => r.Int != 8,
r => (int)r["Int"] != 8);
[ConditionalFact]
public virtual Task NotEqual_with_string()
=> this.TestFilterAsync(
r => r.String != "foo",
r => r["String"] != "foo");
[ConditionalFact]
public virtual Task NotEqual_reversed()
=> this.TestFilterAsync(
r => r.Int != 8,
r => (int)r["Int"] != 8);
[ConditionalFact]
public virtual Task NotEqual_with_null_reference_type()
=> this.TestFilterAsync(
r => r.String != null,
r => r["String"] != null);
[ConditionalFact]
public virtual Task NotEqual_with_null_captured()
{
string? s = null;
return this.TestFilterAsync(
r => r.String != s,
r => r["String"] != s);
}
[ConditionalFact]
public virtual Task Bool()
=> this.TestFilterAsync(
r => r.Bool,
r => (bool)r["Bool"]);
[ConditionalFact]
public virtual Task Bool_And_Bool()
=> this.TestFilterAsync(
r => r.Bool && r.Bool,
r => (bool)r["Bool"] && (bool)r["Bool"]);
[ConditionalFact]
public virtual Task Bool_Or_Not_Bool()
=> this.TestFilterAsync(
r => r.Bool || !r.Bool,
r => (bool)r["Bool"] || !(bool)r["Bool"],
expectAllResults: true);
#endregion Equality
#region Comparison
[ConditionalFact]
public virtual Task GreaterThan_with_int()
=> this.TestFilterAsync(
r => r.Int > 9,
r => (int)r["Int"] > 9);
[ConditionalFact]
public virtual Task GreaterThanOrEqual_with_int()
=> this.TestFilterAsync(
r => r.Int >= 9,
r => (int)r["Int"] >= 9);
[ConditionalFact]
public virtual Task LessThan_with_int()
=> this.TestFilterAsync(
r => r.Int < 10,
r => (int)r["Int"] < 10);
[ConditionalFact]
public virtual Task LessThanOrEqual_with_int()
=> this.TestFilterAsync(
r => r.Int <= 10,
r => (int)r["Int"] <= 10);
#endregion Comparison
#region Logical operators
[ConditionalFact]
public virtual Task And()
=> this.TestFilterAsync(
r => r.Int == 8 && r.String == "foo",
r => (int)r["Int"] == 8 && r["String"] == "foo");
[ConditionalFact]
public virtual Task Or()
=> this.TestFilterAsync(
r => r.Int == 8 || r.String == "foo",
r => (int)r["Int"] == 8 || r["String"] == "foo");
[ConditionalFact]
public virtual Task And_within_And()
=> this.TestFilterAsync(
r => (r.Int == 8 && r.String == "foo") && r.Int2 == 80,
r => ((int)r["Int"] == 8 && r["String"] == "foo") && (int)r["Int2"] == 80);
[ConditionalFact]
public virtual Task And_within_Or()
=> this.TestFilterAsync(
r => (r.Int == 8 && r.String == "foo") || r.Int2 == 100,
r => ((int)r["Int"] == 8 && r["String"] == "foo") || (int)r["Int2"] == 100);
[ConditionalFact]
public virtual Task Or_within_And()
=> this.TestFilterAsync(
r => (r.Int == 8 || r.Int == 9) && r.String == "foo",
r => ((int)r["Int"] == 8 || (int)r["Int"] == 9) && r["String"] == "foo");
[ConditionalFact]
public virtual Task Not_over_Equal()
// ReSharper disable once NegativeEqualityExpression
=> this.TestFilterAsync(
r => !(r.Int == 8),
r => !((int)r["Int"] == 8));
[ConditionalFact]
public virtual Task Not_over_NotEqual()
// ReSharper disable once NegativeEqualityExpression
=> this.TestFilterAsync(
r => !(r.Int != 8),
r => !((int)r["Int"] != 8));
[ConditionalFact]
public virtual Task Not_over_And()
=> this.TestFilterAsync(
r => !(r.Int == 8 && r.String == "foo"),
r => !((int)r["Int"] == 8 && r["String"] == "foo"));
[ConditionalFact]
public virtual Task Not_over_Or()
=> this.TestFilterAsync(
r => !(r.Int == 8 || r.String == "foo"),
r => !((int)r["Int"] == 8 || r["String"] == "foo"));
[ConditionalFact]
public virtual Task Not_over_bool()
=> this.TestFilterAsync(
r => !r.Bool,
r => !(bool)r["Bool"]);
[ConditionalFact]
public virtual Task Not_over_bool_And_Comparison()
=> this.TestFilterAsync(
r => !r.Bool && r.Int != int.MaxValue,
r => !(bool)r["Bool"] && (int)r["Int"] != int.MaxValue);
#endregion Logical operators
#region Contains
[ConditionalFact]
public virtual Task Contains_over_field_string_array()
=> this.TestFilterAsync(
r => r.StringArray.Contains("x"),
r => ((string[])r["StringArray"]!).Contains("x"));
[ConditionalFact]
public virtual Task Contains_over_field_string_List()
=> this.TestFilterAsync(
r => r.StringList.Contains("x"),
r => ((List<string>)r["StringList"]!).Contains("x"));
[ConditionalFact]
public virtual Task Contains_over_inline_int_array()
=> this.TestFilterAsync(
r => new[] { 8, 10 }.Contains(r.Int),
r => new[] { 8, 10 }.Contains((int)r["Int"]));
[ConditionalFact]
public virtual Task Contains_over_inline_string_array()
=> this.TestFilterAsync(
r => new[] { "foo", "baz", "unknown" }.Contains(r.String),
r => new[] { "foo", "baz", "unknown" }.Contains(r["String"]));
[ConditionalFact]
public virtual Task Contains_over_inline_string_array_with_weird_chars()
=> this.TestFilterAsync(
r => new[] { "foo", "baz", "un , ' \"" }.Contains(r.String),
r => new[] { "foo", "baz", "un , ' \"" }.Contains(r["String"]));
[ConditionalFact]
public virtual Task Contains_over_captured_string_array()
{
var array = new[] { "foo", "baz", "unknown" };
return this.TestFilterAsync(
r => array.Contains(r.String),
r => array.Contains(r["String"]));
}
#pragma warning disable RCS1196 // Call extension method as instance method
// C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans");
// this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above).
// See https://github.com/dotnet/runtime/issues/109757 for more context.
// The following tests the various Contains variants directly, without using extension syntax, to ensure everything's
// properly supported.
[ConditionalFact]
public virtual Task Contains_with_Enumerable_Contains()
=> this.TestFilterAsync(
r => Enumerable.Contains(r.StringArray, "x"),
r => ((string[])r["StringArray"]!).Contains("x"));
#if !NETFRAMEWORK
[ConditionalFact]
public virtual Task Contains_with_MemoryExtensions_Contains()
=> this.TestFilterAsync(
r => MemoryExtensions.Contains(r.StringArray, "x"),
r => ((string[])r["StringArray"]!).Contains("x"));
#endif
#if NET10_0_OR_GREATER
[ConditionalFact]
public virtual Task Contains_with_MemoryExtensions_Contains_with_null_comparer()
=> this.TestFilterAsync(
r => MemoryExtensions.Contains(r.StringArray, "x", comparer: null),
r => ((string[])r["StringArray"]!).Contains("x"));
#endif
#pragma warning restore RCS1196 // Call extension method as instance method
[ConditionalFact]
public virtual Task Any_with_Contains_over_inline_string_array()
=> this.TestFilterAsync(
r => r.StringArray.Any(s => new[] { "x", "z", "nonexistent" }.Contains(s)),
r => ((string[])r["StringArray"]!).Any(s => new[] { "x", "z", "nonexistent" }.Contains(s)));
[ConditionalFact]
public virtual Task Any_with_Contains_over_captured_string_array()
{
string[] tagsToFind = ["x", "z", "nonexistent"];
return this.TestFilterAsync(
r => r.StringArray.Any(s => tagsToFind.Contains(s)),
r => ((string[])r["StringArray"]!).Any(s => tagsToFind.Contains(s)));
}
[ConditionalFact]
public virtual Task Any_with_Contains_over_captured_string_list()
{
List<string> tagsToFind = ["x", "z", "nonexistent"];
return this.TestFilterAsync(
r => r.StringArray.Any(s => tagsToFind.Contains(s)),
r => ((string[])r["StringArray"]!).Any(s => tagsToFind.Contains(s)));
}
[ConditionalFact]
public virtual Task Any_over_List_with_Contains_over_captured_string_array()
{
string[] tagsToFind = ["x", "z", "nonexistent"];
return this.TestFilterAsync(
r => r.StringList.Any(s => tagsToFind.Contains(s)),
r => ((List<string>)r["StringList"]!).Any(s => tagsToFind.Contains(s)));
}
#endregion Contains
#region Variable types
[ConditionalFact]
public virtual Task Captured_local_variable()
{
// ReSharper disable once ConvertToConstant.Local
var i = 8;
return this.TestFilterAsync(
r => r.Int == i,
r => (int)r["Int"] == i);
}
[ConditionalFact]
public virtual Task Member_field()
=> this.TestFilterAsync(
r => r.Int == this._memberField,
r => (int)r["Int"] == this._memberField);
[ConditionalFact]
public virtual Task Member_readonly_field()
=> this.TestFilterAsync(
r => r.Int == this._memberReadOnlyField,
r => (int)r["Int"] == this._memberReadOnlyField);
[ConditionalFact]
public virtual Task Member_static_field()
=> this.TestFilterAsync(
r => r.Int == _staticMemberField,
r => (int)r["Int"] == _staticMemberField);
[ConditionalFact]
public virtual Task Member_static_readonly_field()
=> this.TestFilterAsync(
r => r.Int == _staticMemberReadOnlyField,
r => (int)r["Int"] == _staticMemberReadOnlyField);
[ConditionalFact]
public virtual Task Member_nested_access()
=> this.TestFilterAsync(
r => r.Int == this._someWrapper.SomeWrappedValue,
r => (int)r["Int"] == this._someWrapper.SomeWrappedValue);
#pragma warning disable RCS1169 // Make field read-only
#pragma warning disable IDE0044 // Make field read-only
#pragma warning disable RCS1187 // Use constant instead of field
#pragma warning disable CA1802 // Use literals where appropriate
private int _memberField = 8;
private readonly int _memberReadOnlyField = 8;
private static int _staticMemberField = 8;
private static readonly int _staticMemberReadOnlyField = 8;
private SomeWrapper _someWrapper = new();
#pragma warning restore CA1802
#pragma warning restore RCS1187
#pragma warning restore RCS1169
#pragma warning restore IDE0044
private sealed class SomeWrapper
{
public int SomeWrappedValue = 8;
}
#endregion Variable types
#region Miscellaneous
[ConditionalFact]
public virtual Task True()
=> this.TestFilterAsync(r => true, r => true, expectAllResults: true);
#endregion Miscellaneous
protected virtual async Task TestFilterAsync(
Expression<Func<FilterRecord, bool>> filter,
Expression<Func<Dictionary<string, object?>, bool>> dynamicFilter,
bool expectZeroResults = false,
bool expectAllResults = false)
{
var expected = fixture.TestData.AsQueryable().Where(filter).OrderBy(r => r.Key).ToList();
if (expected.Count == 0 && !expectZeroResults)
{
Assert.Fail("The test returns zero results, and so may be unreliable");
}
else if (expectZeroResults && expected.Count != 0)
{
Assert.Fail($"{nameof(expectZeroResults)} was true, but the test returned {expected.Count} results.");
}
if (expected.Count == fixture.TestData.Count && !expectAllResults)
{
Assert.Fail("The test returns all results, and so may be unreliable");
}
else if (expectAllResults && expected.Count != fixture.TestData.Count)
{
Assert.Fail($"{nameof(expectAllResults)} was true, but the test returned {expected.Count} results instead of the expected {fixture.TestData.Count}.");
}
// Execute the query against the vector store, once using the strongly typed filter
// and once using the dynamic filter
var actual = await fixture.Collection.SearchAsync(
new ReadOnlyMemory<float>([1, 2, 3]),
top: fixture.TestData.Count,
new() { Filter = filter })
.Select(r => r.Record)
.OrderBy(r => r.Key)
.ToListAsync();
if (actual.Count != expected.Count)
{
Assert.Fail($"Expected {expected.Count} results, but got {actual.Count}");
}
foreach (var (e, a) in expected.Zip(actual, (e, a) => (e, a)))
{
fixture.AssertEqualFilterRecord(e, a);
}
if (fixture.TestDynamic)
{
var dynamicActual = await fixture.DynamicCollection.SearchAsync(
new ReadOnlyMemory<float>([1, 2, 3]),
top: fixture.TestData.Count,
new() { Filter = dynamicFilter })
.Select(r => r.Record)
.OrderBy(r => r[nameof(FilterRecord.Key)])
.ToListAsync();
if (dynamicActual.Count != expected.Count)
{
Assert.Fail($"Expected {expected.Count} results, but got {actual.Count}");
}
foreach (var (e, a) in expected.Zip(dynamicActual, (e, a) => (e, a)))
{
fixture.AssertEqualDynamic(e, a);
}
}
}
public class FilterRecord : TestRecord<TKey>
{
public ReadOnlyMemory<float>? Vector { get; set; }
public int Int { get; set; }
public string? String { get; set; }
public bool Bool { get; set; }
public int Int2 { get; set; }
public string[] StringArray { get; set; } = null!;
public List<string> StringList { get; set; } = null!;
}
public abstract class Fixture : VectorStoreCollectionFixture<TKey, FilterRecord>
{
protected override string CollectionNameBase => nameof(FilterTests<int>);
public virtual string SpecialCharactersText => """>with $om[ specia]"chara<ters'and\stuff""";
public virtual VectorStoreCollection<object, Dictionary<string, object?>> DynamicCollection { get; protected set; } = null!;
public virtual bool TestDynamic => true;
public override async Task InitializeAsync()
{
await base.InitializeAsync();
if (this.TestDynamic)
{
this.DynamicCollection = this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition());
}
}
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(FilterRecord.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(FilterRecord.Vector), typeof(ReadOnlyMemory<float>?), 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
},
new VectorStoreDataProperty(nameof(FilterRecord.Int), typeof(int)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(FilterRecord.String), typeof(string)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(FilterRecord.Bool), typeof(bool)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(FilterRecord.Int2), typeof(int)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(FilterRecord.StringArray), typeof(string[])) { IsIndexed = true },
new VectorStoreDataProperty(nameof(FilterRecord.StringList), typeof(List<string>)) { IsIndexed = true }
]
};
protected override List<FilterRecord> BuildTestData()
{
// All records have the same vector - this fixture is about testing criteria filtering only
var vector = new ReadOnlyMemory<float>([1, 2, 3]);
return
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Int = 8,
String = "foo",
Bool = true,
Int2 = 80,
StringArray = ["x", "y"],
StringList = ["x", "y"],
Vector = vector
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Int = 9,
String = "bar",
Bool = false,
Int2 = 90,
StringArray = ["a", "b"],
StringList = ["a", "b"],
Vector = vector
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Int = 9,
String = "foo",
Bool = true,
Int2 = 9,
StringArray = ["x"],
StringList = ["x"],
Vector = vector
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Int = 10,
String = null,
Bool = false,
Int2 = 100,
StringArray = ["x", "y", "z"],
StringList = ["x", "y", "z"],
Vector = vector
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Int = 11,
Bool = true,
String = this.SpecialCharactersText,
Int2 = 101,
StringArray = ["y", "z"],
StringList = ["y", "z"],
Vector = vector
}
];
}
public virtual void AssertEqualFilterRecord(FilterRecord x, FilterRecord y)
{
var definitionProperties = this.CreateRecordDefinition().Properties;
Assert.Equal(x.Key, y.Key);
Assert.Equal(x.Int, y.Int);
Assert.Equal(x.String, y.String);
Assert.Equal(x.Int2, y.Int2);
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.Bool)))
{
Assert.Equal(x.Bool, y.Bool);
}
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringArray)))
{
Assert.Equivalent(x.StringArray, y.StringArray);
}
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringList)))
{
Assert.Equivalent(x.StringList, y.StringList);
}
}
public virtual void AssertEqualDynamic(FilterRecord x, Dictionary<string, object?> y)
{
var definitionProperties = this.CreateRecordDefinition().Properties;
Assert.Equal(x.Key, y["Key"]);
Assert.Equal(x.Int, y["Int"]);
Assert.Equal(x.String, y["String"]);
Assert.Equal(x.Int2, y["Int2"]);
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.Bool)))
{
Assert.Equal(x.Bool, y["Bool"]);
}
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringArray)))
{
Assert.Equivalent(x.StringArray, y["StringArray"]);
}
if (definitionProperties.Any(p => p.Name == nameof(FilterRecord.StringList)))
{
Assert.Equivalent(x.StringList, y["StringList"]);
}
}
// In some databases (Azure AI Search), the data shows up but the filtering index isn't yet updated,
// so filtered searches show empty results. Add a filter to the seed data check below.
protected override Task WaitForDataAsync()
=> this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, r => r.Int > 0);
}
}
@@ -0,0 +1,258 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests;
/// <summary>
/// Base class for common integration tests that should pass for any <see cref="IKeywordHybridSearchable{TRecord}"/>.
/// </summary>
/// <typeparam name="TKey">The type of key to use with the record collection.</typeparam>
public abstract class HybridSearchTests<TKey>(
HybridSearchTests<TKey>.VectorAndStringFixture vectorAndStringFixture,
HybridSearchTests<TKey>.MultiTextFixture multiTextFixture)
where TKey : notnull
{
[ConditionalFact]
public async Task HybridSearchAsync()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
// All records have the same vector, but the third contains Grapes, so searching for
// Grapes should return the third record first.
var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["Grapes"], top: 3).ToListAsync();
// Assert
Assert.Equal(3, results.Count);
Assert.Equal(3, results[0].Record.Code);
}
[ConditionalFact]
public async Task HybridSearchAsync_with_filter()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
// All records have the same vector, but the second contains Oranges, however
// adding the filter should limit the results to only the first.
var results = await vectorAndStringFixture.HybridSearchable
.HybridSearchAsync(
vector,
["Oranges"],
top: 3,
new() { Filter = r => r.Code == 1 })
.ToListAsync();
// Assert
Assert.Equal(1, Assert.Single(results).Record.Code);
}
[ConditionalFact]
public async Task HybridSearchAsync_with_top()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
// All records have the same vector, but the second contains Oranges, so the
// second should be returned first.
var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["Oranges"], top: 1).ToListAsync();
// Assert
Assert.Single(results);
Assert.Equal(2, results[0].Record.Code);
}
[ConditionalFact]
public async Task HybridSearchAsync_with_Skip()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
// All records have the same vector, but the first and third contain healthy,
// so when skipping the first two results, we should get the second record.
var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["healthy"], top: 3, new() { Skip = 2 }).ToListAsync();
// Assert
Assert.Single(results);
Assert.Equal(2, results[0].Record.Code);
}
[ConditionalFact]
public async Task HybridSearchAsync_with_multiple_keywords_ranks_matched_keywords_higher()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
var results = await vectorAndStringFixture.HybridSearchable.HybridSearchAsync(vector, ["tangy", "nourishing"], top: 3).ToListAsync();
// Assert
Assert.Equal(3, results.Count);
Assert.True(results[0].Record.Code.Equals(1) || results[0].Record.Code.Equals(2));
Assert.True(results[1].Record.Code.Equals(1) || results[1].Record.Code.Equals(2));
Assert.Equal(3, results[2].Record.Code);
}
[ConditionalFact]
public async Task HybridSearchAsync_with_multiple_text_properties()
{
// Arrange
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
// Act
var results1 = await multiTextFixture.HybridSearchable
.HybridSearchAsync(vector, ["Apples"], top: 4, new() { AdditionalProperty = r => r.Text2 })
.ToListAsync();
var results2 = await multiTextFixture.HybridSearchable
.HybridSearchAsync(vector, ["Oranges"], top: 4, new() { AdditionalProperty = r => r.Text2 })
.ToListAsync();
// Assert
Assert.Equal(2, results1.Count);
Assert.Equal(2, results1[0].Record.Code);
Assert.Equal(1, results1[1].Record.Code);
Assert.Equal(2, results2.Count);
Assert.Equal(1, results2[0].Record.Code);
Assert.Equal(2, results2[1].Record.Code);
}
[ConditionalFact]
public Task HybridSearchAsync_without_explicitly_specified_property_fails()
=> Assert.ThrowsAsync<InvalidOperationException>(async () =>
await multiTextFixture.HybridSearchable
.HybridSearchAsync(new ReadOnlyMemory<float>([1, 0, 0, 0]), ["Apples"], top: 3)
.ToListAsync());
public sealed class VectorAndStringRecord<TRecordKey> : TestRecord<TKey>
{
public string Text { get; set; } = string.Empty;
public int Code { get; set; }
public ReadOnlyMemory<float> Vector { get; set; }
}
public sealed class MultiTextStringRecord<TRecordKey> : TestRecord<TKey>
{
public string Text1 { get; set; } = string.Empty;
public string Text2 { get; set; } = string.Empty;
public int Code { get; set; }
public ReadOnlyMemory<float> Vector { get; set; }
}
public abstract class VectorAndStringFixture : VectorStoreCollectionFixture<TKey, VectorAndStringRecord<TKey>>
{
protected override string CollectionNameBase => "HybridSearchTests";
public IKeywordHybridSearchable<VectorAndStringRecord<TKey>> HybridSearchable
=> (IKeywordHybridSearchable<VectorAndStringRecord<TKey>>)this.Collection;
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(TKey)),
new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true },
new VectorStoreDataProperty("Code", typeof(int)) { IsIndexed = true },
new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), 4) { IndexKind = this.IndexKind },
]
};
protected override List<VectorAndStringRecord<TKey>> BuildTestData()
{
// All records have the same vector - this fixture is about testing the full text search portion of hybrid search
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
return
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Text = "Apples are a healthy and nourishing snack",
Vector = vector,
Code = 1
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Text = "Oranges are tangy and contain vitamin c",
Vector = vector,
Code = 2
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Text = "Grapes are healthy, sweet and juicy",
Vector = vector,
Code = 3
}
];
}
protected override Task WaitForDataAsync()
=> this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorSize: 4);
}
public abstract class MultiTextFixture : VectorStoreCollectionFixture<TKey, MultiTextStringRecord<TKey>>
{
protected override string CollectionNameBase => "MultiTextHybridSearchTests";
public IKeywordHybridSearchable<MultiTextStringRecord<TKey>> HybridSearchable
=> (IKeywordHybridSearchable<MultiTextStringRecord<TKey>>)this.Collection;
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(TKey)),
new VectorStoreDataProperty("Text1", typeof(string)) { IsFullTextIndexed = true },
new VectorStoreDataProperty("Text2", typeof(string)) { IsFullTextIndexed = true },
new VectorStoreDataProperty("Code", typeof(int)) { IsIndexed = true },
new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), 4) { IndexKind = this.IndexKind },
]
};
protected override List<MultiTextStringRecord<TKey>> BuildTestData()
{
// All records have the same vector - this fixture is about testing the full text search portion of hybrid search
var vector = new ReadOnlyMemory<float>([1, 0, 0, 0]);
return
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Text1 = "Apples",
Text2 = "Oranges",
Code = 1,
Vector = vector
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Text1 = "Oranges",
Text2 = "Apples",
Code = 2,
Vector = vector
}
];
}
protected override Task WaitForDataAsync()
=> this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorSize: 4);
}
}
@@ -0,0 +1,88 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests;
public abstract class IndexKindTests<TKey>(IndexKindTests<TKey>.Fixture fixture)
where TKey : notnull
{
[ConditionalFact]
public virtual Task Flat()
=> this.Test(IndexKind.Flat);
protected virtual async Task Test(string indexKind)
{
using var collection = fixture.CreateCollection(indexKind);
await collection.EnsureCollectionDeletedAsync();
await collection.EnsureCollectionExistsAsync();
SearchRecord[] records =
[
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 1,
Vector = new([1, 2, 3]),
},
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 2,
Vector = new([10, 30, 50]),
},
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Int = 3,
Vector = new([100, 40, 70]),
}
];
await collection.UpsertAsync(records);
await fixture.TestStore.WaitForDataAsync(collection, records.Length);
var result = await collection.SearchAsync(new ReadOnlyMemory<float>([10, 30, 50]), top: 1).SingleAsync();
Assert.NotNull(result);
Assert.Equal(2, result.Record.Int);
}
public abstract class Fixture : VectorStoreFixture
{
protected virtual string CollectionNameBase => nameof(IndexKindTests<int>);
public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
protected virtual string? DistanceFunction => null;
public virtual VectorStoreCollection<TKey, SearchRecord> CreateCollection(string indexKind)
{
VectorStoreCollectionDefinition definition = new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(SearchRecord.Key), typeof(TKey)),
new VectorStoreDataProperty(nameof(SearchRecord.Int), typeof(int)),
new VectorStoreVectorProperty(nameof(SearchRecord.Vector), typeof(ReadOnlyMemory<float>), dimensions: 3)
{
IndexKind = indexKind,
DistanceFunction = this.DistanceFunction ?? this.DefaultDistanceFunction
}
]
};
return this.TestStore.CreateCollection<TKey, SearchRecord>(this.CollectionName, definition);
}
}
public class SearchRecord
{
public TKey Key { get; set; } = default!;
public int Int { get; set; }
public ReadOnlyMemory<float> Vector { get; set; }
}
}
@@ -0,0 +1,552 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.ModelTests;
public abstract class BasicModelTests<TKey>(BasicModelTests<TKey>.Fixture fixture) : IAsyncLifetime
where TKey : notnull
{
#region Get
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_single_record(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors });
expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_multiple_records(bool includeVectors)
{
var expectedRecords = fixture.TestData.Take(2);
var ids = expectedRecords.Select(record => record.Key);
var received = await this.Collection.GetAsync(ids, new() { IncludeVectors = includeVectors }).ToArrayAsync();
foreach (var record in expectedRecords)
{
record.AssertEqual(
received.Single(r => r.Key.Equals(record.Key)),
includeVectors,
fixture.TestStore.VectorsComparable);
}
}
[ConditionalFact]
public virtual async Task GetAsync_throws_for_null_key()
{
// Skip this test for value type keys
if (default(TKey) is not null)
{
return;
}
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => this.Collection.GetAsync((TKey)default!));
Assert.Equal("key", ex.ParamName);
}
[ConditionalFact]
public virtual async Task GetAsync_throws_for_null_keys()
{
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => this.Collection.GetAsync(keys: null!).ToArrayAsync().AsTask());
Assert.Equal("keys", ex.ParamName);
}
[ConditionalFact]
public virtual async Task GetAsync_returns_null_for_missing_key()
{
TKey key = fixture.GenerateNextKey<TKey>();
Assert.Null(await this.Collection.GetAsync(key));
}
[ConditionalFact]
public virtual async Task GetAsync_multiple_records_with_missing_keys_returns_only_existing()
{
var expectedRecords = fixture.TestData.Take(2).ToArray();
var missingKey = fixture.GenerateNextKey<TKey>();
var ids = expectedRecords.Select(record => record.Key).Append(missingKey).ToArray();
var received = await this.Collection.GetAsync(ids).ToListAsync();
Assert.Equal(2, received.Count);
foreach (var record in expectedRecords)
{
record.AssertEqual(
received.Single(r => r.Key.Equals(record.Key)),
includeVectors: false,
fixture.TestStore.VectorsComparable);
}
}
[ConditionalFact]
public virtual async Task GetAsync_returns_empty_for_empty_keys()
{
Assert.Empty(await this.Collection.GetAsync([]).ToArrayAsync());
}
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_with_filter(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var results = await this.Collection.GetAsync(
r => r.Number == 1,
top: 2,
new() { IncludeVectors = includeVectors })
.ToListAsync();
var receivedRecord = Assert.Single(results);
expectedRecord.AssertEqual(receivedRecord, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_by_true()
{
Assert.True(fixture.TestData.Count < 100);
var count = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
Assert.Equal(fixture.TestData.Count, count);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_OrderBy()
{
var ascendingNumbers = fixture.TestData.Where(r => r.Number > 1).OrderBy(r => r.Number).Take(2).Select(r => r.Number).ToList();
var descendingNumbers = fixture.TestData.Where(r => r.Number > 1).OrderByDescending(r => r.Number).Take(2).Select(r => r.Number).ToList();
// Make sure the actual results are different for ascending/descending, otherwise the test is meaningless
Assert.NotEqual(ascendingNumbers, descendingNumbers);
var results = await this.Collection.GetAsync(
r => r.Number > 1,
top: 2,
new() { OrderBy = o => o.Ascending(r => r.Number) })
.Select(r => r.Number)
.ToListAsync();
Assert.Equal(ascendingNumbers, results);
results = await this.Collection.GetAsync(
r => r.Number > 1,
top: 2,
new() { OrderBy = o => o.Descending(r => r.Number) })
.Select(r => r.Number)
.ToListAsync();
Assert.Equal(descendingNumbers, results);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_multiple_OrderBys()
{
var ascendingNumbers = fixture.TestData
.OrderByDescending(r => r.Text)
.ThenBy(r => r.Number)
.Take(2).Select(r => r.Number).ToList();
var descendingNumbers = fixture.TestData
.OrderByDescending(r => r.Text)
.ThenByDescending(r => r.Number)
.Take(2)
.Select(r => r.Number)
.ToList();
// Make sure the actual results are different for ascending/descending, otherwise the test is meaningless
Assert.NotEqual(ascendingNumbers, descendingNumbers);
var results = await this.Collection.GetAsync(
r => true,
top: 2,
new() { OrderBy = o => o.Descending(r => r.Text).Ascending(r => r.Number) })
.Select(r => r.Number)
.ToListAsync();
Assert.Equal(ascendingNumbers, results);
results = await this.Collection.GetAsync(
r => true,
top: 2,
new() { OrderBy = o => o.Descending(r => r.Text).Descending(r => r.Number) })
.Select(r => r.Number)
.ToListAsync();
Assert.Equal(descendingNumbers, results);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_OrderBy_and_Skip()
{
var results = await this.Collection.GetAsync(
r => r.Number > 1,
top: 2,
new() { OrderBy = o => o.Ascending(r => r.Number), Skip = 1 })
.Select(r => r.Number)
.ToListAsync();
Assert.Equal(
fixture.TestData.Where(r => r.Number > 1).OrderBy(r => r.Number).Skip(1).Take(2).Select(r => r.Number),
results);
}
#endregion Get
#region Upsert
[ConditionalFact]
public virtual async Task Insert_single_record()
{
TKey expectedKey = fixture.GenerateNextKey<TKey>();
Record inserted = new()
{
Key = expectedKey,
Text = "New record",
Number = 123,
Vector = new([10, 0, 0])
};
Assert.Null(await this.Collection.GetAsync(expectedKey));
await this.Collection.UpsertAsync(inserted);
var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true });
inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable);
await fixture.TestStore.WaitForDataAsync(this.Collection, recordCount: fixture.TestData.Count + 1);
}
[ConditionalFact]
public virtual async Task Update_single_record()
{
var existingRecord = fixture.TestData[1];
Record updated = new()
{
Key = existingRecord.Key,
Text = "Updated record",
Number = 456,
Vector = new([10, 0, 0])
};
Assert.NotNull(await this.Collection.GetAsync(existingRecord.Key));
await this.Collection.UpsertAsync(updated);
var received = await this.Collection.GetAsync(existingRecord.Key, new() { IncludeVectors = true });
updated.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Insert_multiple_records()
{
Record[] newRecords =
[
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Number = 100,
Text = "New record 1",
Vector = new([10, 0, 1])
},
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Number = 101,
Text = "New record 2",
Vector = new([10, 0, 2])
},
];
var keys = newRecords.Select(record => record.Key).ToArray();
Assert.Empty(await this.Collection.GetAsync(keys).ToArrayAsync());
await this.Collection.UpsertAsync(newRecords);
var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync();
Assert.Collection(
received.OrderBy(r => r.Number),
r => newRecords[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable),
r => newRecords[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable));
}
[ConditionalFact]
public virtual async Task Update_multiple_records()
{
Record[] existingRecords =
[
new()
{
Key = fixture.TestData[0].Key,
Number = 101,
Text = "Updated record 1",
Vector = new([10, 0, 1])
},
new()
{
Key = fixture.TestData[1].Key,
Number = 102,
Text = "Updated record 2",
Vector = new([10, 0, 2])
}
];
await this.Collection.UpsertAsync(existingRecords);
var keys = existingRecords.Select(record => record.Key).ToArray();
var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync();
Assert.Collection(
received.OrderBy(r => r.Number),
r => existingRecords[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable),
r => existingRecords[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable));
}
[ConditionalFact]
public virtual async Task Insert_and_update_in_same_batch()
{
Record[] records =
[
new()
{
Key = fixture.GenerateNextKey<TKey>(),
Number = 101,
Text = "New record",
Vector = new([10, 0, 1])
},
new()
{
Key = fixture.TestData[0].Key,
Number = 102,
Text = "Updated record",
Vector = new([10, 0, 2])
},
];
await this.Collection.UpsertAsync(records);
var keys = records.Select(record => record.Key).ToArray();
var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync();
Assert.Collection(
received.OrderBy(r => r.Number),
r => records[0].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable),
r => records[1].AssertEqual(r, includeVectors: true, fixture.TestStore.VectorsComparable));
}
[ConditionalFact]
public virtual async Task UpsertAsync_throws_for_null_batch()
{
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => this.Collection.UpsertAsync(records: null!));
Assert.Equal("records", ex.ParamName);
}
[ConditionalFact]
public virtual async Task UpsertAsync_does_nothing_for_empty_batch()
{
Assert.True(fixture.TestData.Count < 100);
var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
await this.Collection.UpsertAsync([]);
var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
Assert.Equal(afterCount, beforeCount);
}
#endregion Upsert
#region Delete
[ConditionalFact]
public virtual async Task Delete_single_record()
{
var keyToRemove = fixture.TestData[0].Key;
await this.Collection.DeleteAsync(keyToRemove);
Assert.Null(await this.Collection.GetAsync(keyToRemove));
}
[ConditionalFact]
public virtual async Task Delete_multiple_records()
{
TKey[] keysToRemove = [fixture.TestData[0].Key, fixture.TestData[1].Key];
await this.Collection.DeleteAsync(keysToRemove);
Assert.Empty(await this.Collection.GetAsync(keysToRemove).ToArrayAsync());
}
[ConditionalFact]
public virtual async Task DeleteAsync_does_nothing_for_non_existing_key()
{
var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
await this.Collection.DeleteAsync(fixture.GenerateNextKey<TKey>());
var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
Assert.Equal(afterCount, beforeCount);
}
[ConditionalFact]
public virtual async Task DeleteAsync_does_nothing_for_empty_batch()
{
var beforeCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
await this.Collection.DeleteAsync([]);
var afterCount = await this.Collection.GetAsync(r => true, top: 100).CountAsync();
Assert.Equal(afterCount, beforeCount);
}
[ConditionalFact]
public virtual async Task DeleteAsync_throws_for_null_keys()
{
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => this.Collection.DeleteAsync(keys: null!));
Assert.Equal("keys", ex.ParamName);
}
#endregion Delete
#region Search
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task SearchAsync(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var result = await this.Collection
.SearchAsync(
expectedRecord.Vector,
top: 1,
new() { IncludeVectors = includeVectors })
.SingleAsync();
expectedRecord.AssertEqual(result.Record, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_Skip()
{
var result = await this.Collection
.SearchAsync(
fixture.TestData[0].Vector,
top: 1,
new() { Skip = 1 })
.SingleAsync();
fixture.TestData[1].AssertEqual(result.Record, includeVectors: false, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_Filter()
{
var result = await this.Collection
.SearchAsync(
fixture.TestData[0].Vector,
top: 1,
new() { Filter = r => r.Number == 2 })
.SingleAsync();
fixture.TestData[1].AssertEqual(result.Record, includeVectors: false, fixture.TestStore.VectorsComparable);
}
// For ScoreThreshold, see DistanceFunctionTests (to ensure we tests thresholds for each and every function)
#endregion Search
protected VectorStoreCollection<TKey, Record> Collection => fixture.Collection;
public abstract class Fixture : VectorStoreCollectionFixture<TKey, Record>
{
protected override string CollectionNameBase => nameof(BasicModelTests<int>);
protected override List<Record> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Number = 1,
Text = "foo",
Vector = new([1, 2, 3])
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Number = 2,
Text = "bar",
Vector = new([1, 2, 4])
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Number = 3,
Text = "foo", // identical text as above
Vector = new([1, 2, 5])
}
];
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(Record.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(Record.Vector), typeof(ReadOnlyMemory<float>), 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
},
new VectorStoreDataProperty(nameof(Record.Number), typeof(int)) { IsIndexed = true },
new VectorStoreDataProperty(nameof(Record.Text), typeof(string)) { IsIndexed = true },
]
};
}
public sealed class Record : TestRecord<TKey>
{
[VectorStoreData(StorageName = "text")]
public string? Text { get; set; }
[VectorStoreData(StorageName = "number")]
public int Number { get; set; }
[VectorStoreVector(Dimensions: 3, StorageName = "vector")]
public ReadOnlyMemory<float> Vector { get; set; }
public void AssertEqual(Record? other, bool includeVectors, bool compareVectors)
{
Assert.NotNull(other);
Assert.Equal(this.Key, other.Key);
Assert.Equal(this.Text, other.Text);
Assert.Equal(this.Number, other.Number);
if (includeVectors)
{
Assert.Equal(this.Vector.Span.Length, other.Vector.Span.Length);
if (compareVectors)
{
Assert.Equal(this.Vector.ToArray(), other.Vector.ToArray());
}
}
else
{
Assert.Equal(0, other.Vector.Length);
}
}
public override string ToString()
=> $"Key: {this.Key}, Text: {this.Text}";
}
public Task InitializeAsync()
=> fixture.ReseedAsync();
public Task DisposeAsync()
=> Task.CompletedTask;
public static readonly TheoryData<bool> IncludeVectorsData = [false, true];
}
@@ -0,0 +1,450 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.ModelTests;
public abstract class DynamicModelTests<TKey>(DynamicModelTests<TKey>.Fixture fixture) : IAsyncLifetime
where TKey : notnull
{
#region Get
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_single_record(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var received = await fixture.Collection.GetAsync(
(TKey)expectedRecord[KeyPropertyName]!,
new() { IncludeVectors = includeVectors });
AssertEquivalent(expectedRecord, received, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_multiple_records(bool includeVectors)
{
var expectedRecords = fixture.TestData.Take(2);
var ids = expectedRecords.Select(record => record[KeyPropertyName]!);
var received = await fixture.Collection.GetAsync(ids, new() { IncludeVectors = includeVectors }).ToArrayAsync();
foreach (var record in expectedRecords)
{
AssertEquivalent(
record,
received.Single(r => r[KeyPropertyName]!.Equals(record[KeyPropertyName])),
includeVectors,
fixture.TestStore.VectorsComparable);
}
}
[ConditionalFact]
public virtual async Task GetAsync_throws_for_null_key()
{
// Skip this test for value type keys
if (default(TKey) is not null)
{
return;
}
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => fixture.Collection.GetAsync((TKey)default!));
Assert.Equal("key", ex.ParamName);
}
[ConditionalFact]
public virtual async Task GetAsync_throws_for_null_keys()
{
ArgumentNullException ex = await Assert.ThrowsAsync<ArgumentNullException>(() => fixture.Collection.GetAsync(keys: null!).ToArrayAsync().AsTask());
Assert.Equal("keys", ex.ParamName);
}
[ConditionalFact]
public virtual async Task GetAsync_returns_null_for_missing_key()
{
TKey key = fixture.GenerateNextKey<TKey>();
Assert.Null(await fixture.Collection.GetAsync(key));
}
[ConditionalFact]
public virtual async Task GetAsync_returns_empty_for_empty_keys()
{
Assert.Empty(await fixture.Collection.GetAsync([]).ToArrayAsync());
}
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_with_filter(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var results = await fixture.Collection.GetAsync(
r => (int)r[IntegerPropertyName]! == 1,
top: 2,
new() { IncludeVectors = includeVectors })
.ToListAsync();
var receivedRecord = Assert.Single(results);
AssertEquivalent(expectedRecord, receivedRecord, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_by_true()
{
var count = await fixture.Collection.GetAsync(r => true, top: 100).CountAsync();
Assert.Equal(fixture.TestData.Count, count);
Assert.True(count < 100);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_OrderBy()
{
var ascendingNumbers = fixture.TestData
.Where(r => (int)r[IntegerPropertyName]! > 1)
.OrderBy(r => r[IntegerPropertyName])
.Take(2)
.Select(r => (int)r[IntegerPropertyName]!)
.ToList();
var descendingNumbers = fixture.TestData
.Where(r => (int)r[IntegerPropertyName]! > 1)
.OrderByDescending(r => r[IntegerPropertyName])
.Take(2)
.Select(r => (int)r[IntegerPropertyName]!)
.ToList();
// Make sure the actual results are different for ascending/descending, otherwise the test is meaningless
Assert.NotEqual(ascendingNumbers, descendingNumbers);
// Finally, query once with ascending and once with descending, comparing against the expected results above.
var results = await fixture.Collection.GetAsync(
r => (int)r[IntegerPropertyName]! > 1,
top: 2,
new() { OrderBy = o => o.Ascending(r => r[IntegerPropertyName]) })
.Select(r => (int)r[IntegerPropertyName]!)
.ToListAsync();
Assert.Equal(ascendingNumbers, results);
results = await fixture.Collection.GetAsync(
r => (int)r[IntegerPropertyName]! > 1,
top: 2,
new() { OrderBy = o => o.Descending(r => r[IntegerPropertyName]) })
.Select(r => (int)r[IntegerPropertyName]!)
.ToListAsync();
Assert.Equal(descendingNumbers, results);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_multiple_OrderBys()
{
var ascendingNumbers = fixture.TestData
.OrderByDescending(r => r[StringPropertyName])
.ThenBy(r => r[IntegerPropertyName])
.Take(2)
.Select(r => (int)r[IntegerPropertyName]!)
.ToList();
var descendingNumbers = fixture.TestData
.OrderByDescending(r => r[StringPropertyName])
.ThenByDescending(r => r[IntegerPropertyName])
.Take(2)
.Select(r => (int)r[IntegerPropertyName]!)
.ToList();
// Make sure the actual results are different for ascending/descending, otherwise the test is meaningless
Assert.NotEqual(ascendingNumbers, descendingNumbers);
var results = await fixture.Collection.GetAsync(
r => true,
top: 2,
new() { OrderBy = o => o.Descending(r => r[StringPropertyName]).Ascending(r => r[IntegerPropertyName]) })
.Select(r => (int)r[IntegerPropertyName]!)
.ToListAsync();
Assert.Equal(ascendingNumbers, results);
results = await fixture.Collection.GetAsync(
r => true,
top: 2,
new() { OrderBy = o => o.Descending(r => r[StringPropertyName]).Descending(r => r[IntegerPropertyName]) })
.Select(r => (int)r[IntegerPropertyName]!)
.ToListAsync();
Assert.Equal(descendingNumbers, results);
}
[ConditionalFact]
public virtual async Task GetAsync_with_filter_and_OrderBy_and_Skip()
{
var results = await fixture.Collection.GetAsync(
r => (int)r[IntegerPropertyName]! > 1,
top: 2,
new() { OrderBy = o => o.Ascending(r => r[IntegerPropertyName]), Skip = 1 })
.Select(r => (int)r[IntegerPropertyName]!)
.ToListAsync();
Assert.Equal(
fixture.TestData
.Where(r => (int)r[IntegerPropertyName]! > 1)
.OrderBy(r => r[IntegerPropertyName])
.Skip(1)
.Take(2)
.Select(r => (int)r[IntegerPropertyName]!),
results);
}
#endregion Get
#region Upsert
[ConditionalFact]
public virtual async Task Insert_single_record()
{
TKey expectedKey = fixture.GenerateNextKey<TKey>();
var inserted = new Dictionary<string, object?>
{
[KeyPropertyName] = expectedKey,
[StringPropertyName] = "some",
[IntegerPropertyName] = 123,
[VectorPropertyName] = new ReadOnlyMemory<float>([10, 0, 0])
};
Assert.Null(await this.Collection.GetAsync(expectedKey));
await this.Collection.UpsertAsync(inserted);
var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true });
AssertEquivalent(inserted, received, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Update_single_record()
{
var existingRecord = fixture.TestData[1];
var updated = new Dictionary<string, object?>
{
[KeyPropertyName] = existingRecord[KeyPropertyName],
[StringPropertyName] = "different",
[IntegerPropertyName] = 456,
[VectorPropertyName] = new ReadOnlyMemory<float>(Enumerable.Repeat(0.7f, 3).ToArray())
};
Assert.NotNull(await this.Collection.GetAsync((TKey)existingRecord[KeyPropertyName]!));
await this.Collection.UpsertAsync(updated);
var received = await this.Collection.GetAsync((TKey)existingRecord[KeyPropertyName]!, new() { IncludeVectors = true });
AssertEquivalent(updated, received, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Insert_multiple_records()
{
Dictionary<string, object?>[] newRecords =
[
new()
{
[KeyPropertyName] = fixture.GenerateNextKey<TKey>(),
[IntegerPropertyName] = 100,
[StringPropertyName] = "New record 1",
[VectorPropertyName] = new ReadOnlyMemory<float>([10, 0, 1])
},
new()
{
[KeyPropertyName] = fixture.GenerateNextKey<TKey>(),
[IntegerPropertyName] = 101,
[StringPropertyName] = "New record 2",
[VectorPropertyName] = new ReadOnlyMemory<float>([10, 0, 2])
},
];
var keys = newRecords.Select(record => record[KeyPropertyName]!).ToArray();
Assert.Empty(await this.Collection.GetAsync(keys).ToArrayAsync());
await this.Collection.UpsertAsync(newRecords);
var received = await this.Collection.GetAsync(keys, new() { IncludeVectors = true }).ToArrayAsync();
Assert.Collection(
received.OrderBy(r => r[IntegerPropertyName]),
r => AssertEquivalent(newRecords[0], r, includeVectors: true, fixture.TestStore.VectorsComparable),
r => AssertEquivalent(newRecords[1], r, includeVectors: true, fixture.TestStore.VectorsComparable));
}
#endregion Upsert
#region Delete
[ConditionalFact]
public async Task Delete_single_record()
{
var recordToRemove = fixture.TestData[2];
Assert.NotNull(await fixture.Collection.GetAsync((TKey)recordToRemove[KeyPropertyName]!));
await fixture.Collection.DeleteAsync((TKey)recordToRemove[KeyPropertyName]!);
Assert.Null(await fixture.Collection.GetAsync((TKey)recordToRemove[KeyPropertyName]!));
}
// TODO: https://github.com/microsoft/semantic-kernel/issues/13303
// [ConditionalFact]
// public virtual async Task Delete_multiple_records()
// {
// TKey[] keysToRemove = [(TKey)fixture.TestData[0][KeyPropertyName]!, (TKey)fixture.TestData[1][KeyPropertyName]!];
// await this.Collection.DeleteAsync(keysToRemove);
// Assert.Empty(await this.Collection.GetAsync(keysToRemove).ToArrayAsync());
// Assert.Equal(fixture.TestData.Count - 2, await this.GetRecordCount());
// }
[ConditionalFact]
public virtual async Task DeleteAsync_does_nothing_for_non_existing_key()
{
TKey key = fixture.GenerateNextKey<TKey>();
await fixture.Collection.DeleteAsync(key);
}
#endregion Delete
#region Search
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task SearchAsync(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var result = await this.Collection
.SearchAsync(
expectedRecord[VectorPropertyName]!,
top: 1,
new() { IncludeVectors = includeVectors })
.SingleAsync();
AssertEquivalent(expectedRecord, result.Record, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_Skip()
{
var result = await this.Collection
.SearchAsync(
fixture.TestData[0][VectorPropertyName]!,
top: 1,
new() { Skip = 1 })
.SingleAsync();
AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: false, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task SearchAsync_with_Filter()
{
var result = await this.Collection
.SearchAsync(
fixture.TestData[0][VectorPropertyName]!,
top: 1,
new() { Filter = r => (int)r[IntegerPropertyName]! == 2 })
.SingleAsync();
AssertEquivalent(fixture.TestData[1], result.Record, includeVectors: false, fixture.TestStore.VectorsComparable);
}
#endregion Search
protected static void AssertEquivalent(Dictionary<string, object?> expected, Dictionary<string, object?>? actual, bool includeVectors, bool compareVectors)
{
Assert.NotNull(actual);
Assert.Equal(expected[KeyPropertyName], actual[KeyPropertyName]);
Assert.Equal(expected[StringPropertyName], actual[StringPropertyName]);
Assert.Equal(expected[IntegerPropertyName], actual[IntegerPropertyName]);
if (includeVectors)
{
Assert.Equal(
((ReadOnlyMemory<float>)expected[VectorPropertyName]!).Length,
((ReadOnlyMemory<float>)actual[VectorPropertyName]!).Length);
if (compareVectors)
{
Assert.Equal(
((ReadOnlyMemory<float>)expected[VectorPropertyName]!).ToArray(),
((ReadOnlyMemory<float>)actual[VectorPropertyName]!).ToArray());
}
}
else
{
Assert.False(actual.ContainsKey(VectorPropertyName));
}
}
public const string KeyPropertyName = "key";
public const string StringPropertyName = "text";
public const string IntegerPropertyName = "integer";
public const string VectorPropertyName = "vector";
protected VectorStoreCollection<object, Dictionary<string, object?>> Collection => fixture.Collection;
public abstract class Fixture : DynamicVectorStoreCollectionFixture<TKey>
{
protected override string CollectionNameBase => nameof(DynamicModelTests<int>);
protected override string KeyPropertyName => DynamicModelTests<TKey>.KeyPropertyName;
protected override VectorStoreCollection<object, Dictionary<string, object?>> GetCollection()
=> this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition());
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(this.KeyPropertyName, typeof(TKey)),
new VectorStoreDataProperty(StringPropertyName, typeof(string)) { IsIndexed = true},
new VectorStoreDataProperty(IntegerPropertyName, typeof(int)) { IsIndexed = true },
new VectorStoreVectorProperty(VectorPropertyName, typeof(ReadOnlyMemory<float>), dimensions: 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
}
]
};
protected override List<Dictionary<string, object?>> BuildTestData() =>
[
new()
{
[this.KeyPropertyName] = this.GenerateNextKey<TKey>(),
[StringPropertyName] = "foo",
[IntegerPropertyName] = 1,
[VectorPropertyName] = new ReadOnlyMemory<float>([1, 2, 3])
},
new()
{
[this.KeyPropertyName] = this.GenerateNextKey<TKey>(),
[StringPropertyName] = "bar",
[IntegerPropertyName] = 2,
[VectorPropertyName] = new ReadOnlyMemory<float>([1, 2, 4])
},
new()
{
[this.KeyPropertyName] = this.GenerateNextKey<TKey>(),
[StringPropertyName] = "foo", // identical text as above
[IntegerPropertyName] = 3,
[VectorPropertyName] = new ReadOnlyMemory<float>([1, 2, 5])
}
];
}
public Task InitializeAsync()
=> fixture.ReseedAsync();
public Task DisposeAsync()
=> Task.CompletedTask;
public static readonly TheoryData<bool> IncludeVectorsData = [false, true];
}
@@ -0,0 +1,162 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.ModelTests;
/// <summary>
/// Tests using a model with multiple vectors.
/// </summary>
public class MultiVectorModelTests<TKey>(MultiVectorModelTests<TKey>.Fixture fixture) : IAsyncLifetime
where TKey : notnull
{
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_single_record(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors });
expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Insert_single_record()
{
TKey expectedKey = fixture.GenerateNextKey<TKey>();
MultiVectorRecord inserted = new()
{
Key = expectedKey,
Number = 10,
Vector1 = new([10, 0, 0]),
Vector2 = new([10, 0, 0]),
};
Assert.Null(await this.Collection.GetAsync(expectedKey));
await this.Collection.UpsertAsync(inserted);
var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true });
inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Delete_single_record()
{
var keyToRemove = fixture.TestData[0].Key;
await this.Collection.DeleteAsync(keyToRemove);
Assert.Null(await this.Collection.GetAsync(keyToRemove));
}
[ConditionalFact]
public virtual async Task SearchAsync_with_multiple_vector_properties()
{
var result = await this.Collection
.SearchAsync(new ReadOnlyMemory<float>([1, 2, 3]), top: 1, new() { VectorProperty = r => r.Vector1, IncludeVectors = true })
.SingleAsync();
fixture.TestData[0].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable);
result = await this.Collection
.SearchAsync(new ReadOnlyMemory<float>([10, 2, 6]), top: 1, new() { VectorProperty = r => r.Vector2, IncludeVectors = true })
.SingleAsync();
fixture.TestData[1].AssertEqual(result.Record, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Search_without_explicitly_specified_vector_property_fails()
{
var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
await this.Collection.SearchAsync(new ReadOnlyMemory<float>([1, 2, 3]), top: 1).ToListAsync());
Assert.Equal($"The '{nameof(MultiVectorRecord)}' type has multiple vector properties, please specify your chosen property via options.", exception.Message);
}
protected VectorStoreCollection<TKey, MultiVectorRecord> Collection => fixture.Collection;
public abstract class Fixture : VectorStoreCollectionFixture<TKey, MultiVectorRecord>
{
protected override string CollectionNameBase => "MultiVectorModelTests";
protected override List<MultiVectorRecord> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Number = 1,
Vector1 = new([1, 2, 3]),
Vector2 = new([10, 2, 4])
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Number = 2,
Vector1 = new([1, 2, 5]),
Vector2 = new([10, 2, 6])
}
];
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(MultiVectorRecord.Key), typeof(TKey)),
new VectorStoreDataProperty(nameof(MultiVectorRecord.Number), typeof(int)),
new VectorStoreVectorProperty(nameof(MultiVectorRecord.Vector1), typeof(ReadOnlyMemory<float>), 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
},
new VectorStoreVectorProperty(nameof(MultiVectorRecord.Vector2), typeof(ReadOnlyMemory<float>), 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
}
]
};
protected override Task WaitForDataAsync()
=> this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count, vectorProperty: r => r.Vector1);
}
public sealed class MultiVectorRecord : TestRecord<TKey>
{
public int Number { get; set; }
public ReadOnlyMemory<float> Vector1 { get; set; }
public ReadOnlyMemory<float> Vector2 { get; set; }
public void AssertEqual(MultiVectorRecord? other, bool includeVectors, bool compareVectors)
{
Assert.NotNull(other);
Assert.Equal(this.Key, other.Key);
Assert.Equal(this.Number, other.Number);
if (includeVectors)
{
Assert.Equal(this.Vector1.Span.Length, other.Vector1.Span.Length);
Assert.Equal(this.Vector2.Span.Length, other.Vector2.Span.Length);
if (compareVectors)
{
Assert.True(this.Vector1.Span.SequenceEqual(other.Vector1.Span));
Assert.True(this.Vector2.Span.SequenceEqual(other.Vector2.Span));
}
}
}
}
public Task InitializeAsync()
=> fixture.ReseedAsync();
public Task DisposeAsync()
=> Task.CompletedTask;
public static readonly TheoryData<bool> IncludeVectorsData = [false, true];
}
@@ -0,0 +1,115 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.ModelTests;
/// <summary>
/// Tests using a model without data fields, only a key and an embedding.
/// </summary>
public class NoDataModelTests<TKey>(NoDataModelTests<TKey>.Fixture fixture) : IAsyncLifetime
where TKey : notnull
{
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_single_record(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors });
expectedRecord.AssertEqual(received, includeVectors, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Insert_single_record()
{
TKey expectedKey = fixture.GenerateNextKey<TKey>();
NoDataRecord inserted = new()
{
Key = expectedKey,
Floats = new([10, 0, 0])
};
Assert.Null(await this.Collection.GetAsync(expectedKey));
await this.Collection.UpsertAsync(inserted);
var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true });
inserted.AssertEqual(received, includeVectors: true, fixture.TestStore.VectorsComparable);
}
[ConditionalFact]
public virtual async Task Delete_single_record()
{
var keyToRemove = fixture.TestData[0].Key;
await this.Collection.DeleteAsync(keyToRemove);
Assert.Null(await this.Collection.GetAsync(keyToRemove));
}
protected VectorStoreCollection<TKey, NoDataRecord> Collection => fixture.Collection;
public abstract class Fixture : VectorStoreCollectionFixture<TKey, NoDataRecord>
{
protected override string CollectionNameBase => nameof(NoDataModelTests<int>);
protected override List<NoDataRecord> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Floats = new([1, 2, 3])
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Floats = new([1, 2, 4])
}
];
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(NoDataRecord.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(NoDataRecord.Floats), typeof(ReadOnlyMemory<float>), 3)
{
IndexKind = this.IndexKind
}
]
};
}
public sealed class NoDataRecord : TestRecord<TKey>
{
[VectorStoreVector(Dimensions: 3, StorageName = "embedding")]
public ReadOnlyMemory<float> Floats { get; set; }
public void AssertEqual(NoDataRecord? other, bool includeVectors, bool compareVectors)
{
Assert.NotNull(other);
Assert.Equal(this.Key, other.Key);
if (includeVectors)
{
Assert.Equal(this.Floats.Span.Length, other.Floats.Span.Length);
if (compareVectors)
{
Assert.True(this.Floats.Span.SequenceEqual(other.Floats.Span));
}
}
}
}
public Task InitializeAsync()
=> fixture.ReseedAsync();
public Task DisposeAsync()
=> Task.CompletedTask;
public static readonly TheoryData<bool> IncludeVectorsData = [false, true];
}
@@ -0,0 +1,121 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.ModelTests;
/// <summary>
/// Tests operations using a model without a vector.
/// This is only supported by a subset of databases so only extend if applicable for your database.
/// </summary>
public class NoVectorModelTests<TKey>(NoVectorModelTests<TKey>.Fixture fixture) : IAsyncLifetime
where TKey : notnull
{
[ConditionalTheory, MemberData(nameof(IncludeVectorsData))]
public virtual async Task GetAsync_single_record(bool includeVectors)
{
var expectedRecord = fixture.TestData[0];
var received = await this.Collection.GetAsync(expectedRecord.Key, new() { IncludeVectors = includeVectors });
expectedRecord.AssertEqual(received);
}
[ConditionalFact]
public virtual async Task Insert_single_record()
{
TKey expectedKey = fixture.GenerateNextKey<TKey>();
NoVectorRecord inserted = new()
{
Key = expectedKey,
Text = "New record"
};
Assert.Null(await this.Collection.GetAsync(expectedKey));
await this.Collection.UpsertAsync(inserted);
var received = await this.Collection.GetAsync(expectedKey, new() { IncludeVectors = true });
inserted.AssertEqual(received);
}
[ConditionalFact]
public virtual async Task Delete_single_record()
{
var keyToRemove = fixture.TestData[0].Key;
await this.Collection.DeleteAsync(keyToRemove);
Assert.Null(await this.Collection.GetAsync(keyToRemove));
}
protected VectorStoreCollection<TKey, NoVectorRecord> Collection => fixture.Collection;
public abstract class Fixture : VectorStoreCollectionFixture<TKey, NoVectorRecord>
{
protected override string CollectionNameBase => nameof(NoVectorModelTests<int>);
protected override List<NoVectorRecord> BuildTestData() =>
[
new()
{
Key = this.GenerateNextKey<TKey>(),
Text = "foo",
},
new()
{
Key = this.GenerateNextKey<TKey>(),
Text = "bar",
}
];
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(NoVectorRecord.Key), typeof(TKey)),
new VectorStoreDataProperty(nameof(NoVectorRecord.Text), typeof(string)) { IsIndexed = true }
]
};
// The default implementation of WaitForDataAsync uses SearchAsync, but our model has no vectors.
protected override async Task WaitForDataAsync()
{
for (var i = 0; i < 200; i++)
{
var results = await this.Collection.GetAsync([this.TestData[0].Key, this.TestData[1].Key]).ToArrayAsync();
if (results.Length == this.TestData.Count && results.All(r => r != null))
{
return;
}
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
throw new InvalidOperationException("Data did not appear in the collection within the expected time.");
}
}
public sealed class NoVectorRecord : TestRecord<TKey>
{
[VectorStoreData(StorageName = "text")]
public string? Text { get; set; }
public void AssertEqual(NoVectorRecord? other)
{
Assert.NotNull(other);
Assert.Equal(this.Key, other.Key);
Assert.Equal(this.Text, other.Text);
}
}
public Task InitializeAsync()
=> fixture.ReseedAsync();
public Task DisposeAsync()
=> Task.CompletedTask;
public static readonly TheoryData<bool> IncludeVectorsData = [false, true];
}
@@ -0,0 +1,28 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VectorData.ConformanceTests.Support;
public abstract class DynamicVectorStoreCollectionFixture<TKey> : VectorStoreCollectionFixtureBase<object, Dictionary<string, object?>>
where TKey : notnull
{
protected abstract string KeyPropertyName { get; }
public virtual async Task ReseedAsync()
{
// TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830
const int BatchSize = 100;
List<TKey> keys = [];
do
{
await foreach (var record in this.Collection.GetAsync(r => true, top: BatchSize))
{
// TODO: We don't use batching delete because of https://github.com/microsoft/semantic-kernel/issues/13303
await this.Collection.DeleteAsync((TKey)record[this.KeyPropertyName]!);
}
} while (keys.Count == BatchSize);
await this.SeedAsync();
}
}
@@ -0,0 +1,11 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
namespace VectorData.ConformanceTests.Support;
public abstract class TestRecord<TKey>
{
[VectorStoreKey]
public TKey Key { get; set; } = default!;
}
@@ -0,0 +1,157 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Globalization;
using System.Linq.Expressions;
using Microsoft.Extensions.VectorData;
namespace VectorData.ConformanceTests.Support;
#pragma warning disable CA1001 // Type owns disposable fields but is not disposable
public abstract class TestStore
{
private readonly SemaphoreSlim _lock = new(1, 1);
private int _referenceCount;
private VectorStore? _defaultVectorStore;
/// <summary>
/// Some databases modify vectors on upsert, e.g. normalizing them, so vectors
/// returned cannot be compared with the original ones.
/// </summary>
public virtual bool VectorsComparable => true;
/// <summary>
/// Whether the database supports filtering by score threshold in vector search.
/// </summary>
public virtual bool SupportsScoreThreshold => true;
public virtual string DefaultDistanceFunction => DistanceFunction.CosineSimilarity;
public virtual string DefaultIndexKind => IndexKind.Flat;
protected abstract Task StartAsync();
protected virtual Task StopAsync()
=> Task.CompletedTask;
public VectorStore DefaultVectorStore
{
get => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized");
set => this._defaultVectorStore = value;
}
public virtual async Task ReferenceCountingStartAsync()
{
await this._lock.WaitAsync();
try
{
if (this._referenceCount++ == 0)
{
await this.StartAsync();
}
}
finally
{
this._lock.Release();
}
}
public virtual async Task ReferenceCountingStopAsync()
{
await this._lock.WaitAsync();
try
{
if (--this._referenceCount == 0)
{
await this.StopAsync();
this._defaultVectorStore?.Dispose();
}
}
finally
{
this._lock.Release();
}
}
public virtual TKey GenerateKey<TKey>(int value)
=> typeof(TKey) switch
{
_ when typeof(TKey) == typeof(int) => (TKey)(object)value,
_ when typeof(TKey) == typeof(long) => (TKey)(object)(long)value,
_ when typeof(TKey) == typeof(ulong) => (TKey)(object)(ulong)value,
_ when typeof(TKey) == typeof(string) => (TKey)(object)value.ToString(CultureInfo.InvariantCulture),
_ when typeof(TKey) == typeof(Guid) => (TKey)(object)new Guid($"00000000-0000-0000-0000-00{value:0000000000}"),
_ => throw new NotSupportedException($"Unsupported key of type '{typeof(TKey).Name}', override {nameof(TestStore)}.{nameof(this.GenerateKey)}")
};
/// <summary>
/// Applies any provider-specific rules to collection names (e.g. all-lowercase).
/// </summary>
/// <param name="baseName"></param>
/// <returns></returns>
public virtual string AdjustCollectionName(string baseName)
=> baseName;
/// <summary>
/// Creates a collection for the given name and definition.
/// Override this to provide provider-specific collection options (e.g., partition key configuration).
/// </summary>
public virtual VectorStoreCollection<TKey, TRecord> CreateCollection<TKey, TRecord>(
string name,
VectorStoreCollectionDefinition definition)
where TKey : notnull
where TRecord : class
=> this.DefaultVectorStore.GetCollection<TKey, TRecord>(name, definition);
/// <summary>
/// Creates a dynamic collection for the given name and definition.
/// Override this to provide provider-specific collection options (e.g., partition key configuration).
/// </summary>
public virtual VectorStoreCollection<object, Dictionary<string, object?>> CreateDynamicCollection(
string name,
VectorStoreCollectionDefinition definition)
=> this.DefaultVectorStore.GetDynamicCollection(name, definition);
/// <summary>Loops until the expected number of records is visible in the given collection.</summary>
/// <remarks>Some databases upsert asynchronously, meaning that our seed data may not be visible immediately to tests.</remarks>
public virtual async Task WaitForDataAsync<TKey, TRecord>(
VectorStoreCollection<TKey, TRecord> collection,
int recordCount,
Expression<Func<TRecord, bool>>? filter = null,
Expression<Func<TRecord, object?>>? vectorProperty = null,
int? vectorSize = null,
object? dummyVector = null)
where TKey : notnull
where TRecord : class
{
if (vectorSize is not null && dummyVector is not null)
{
throw new ArgumentException("vectorSize or dummyVector can't both be set");
}
var vector = dummyVector ?? new ReadOnlyMemory<float>(Enumerable.Range(0, vectorSize ?? 3).Select(i => (float)i).ToArray());
for (var i = 0; i < 200; i++)
{
// Note that we very intentionally use SearchAsync and not filtering GetAsync, as we want to wait until the data is visible
// specifically via vector search (some databases may show data via filtering before they are indexed for vector search).
var results = collection.SearchAsync(
vector,
top: recordCount is 0 ? 1 : recordCount,
new()
{
Filter = filter,
VectorProperty = vectorProperty
});
var count = await results.CountAsync();
if (count == recordCount)
{
return;
}
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
throw new InvalidOperationException("Data did not appear in the collection within the expected time.");
}
}
@@ -0,0 +1,24 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VectorData.ConformanceTests.Support;
public abstract class VectorStoreCollectionFixture<TKey, TRecord> : VectorStoreCollectionFixtureBase<TKey, TRecord>
where TKey : notnull
where TRecord : TestRecord<TKey>
{
public virtual async Task ReseedAsync()
{
// TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830
const int BatchSize = 100;
TKey[] keys;
do
{
keys = await this.Collection.GetAsync(r => true, top: BatchSize).Select(r => r.Key).ToArrayAsync();
await this.Collection.DeleteAsync(keys);
} while (keys.Length == BatchSize);
await this.SeedAsync();
}
}
@@ -0,0 +1,69 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
namespace VectorData.ConformanceTests.Support;
#pragma warning disable CA1721 // Property names should not match get methods
/// <summary>
/// A test fixture that sets up a single collection in the test vector store, with a specific record definition
/// and test data.
/// </summary>
public abstract class VectorStoreCollectionFixtureBase<TKey, TRecord> : VectorStoreFixture
where TKey : notnull
where TRecord : class
{
private List<TRecord>? _testData;
public abstract VectorStoreCollectionDefinition CreateRecordDefinition();
protected virtual List<TRecord> BuildTestData() => [];
/// <summary>
/// The base name for the test collection used in tests, before any provider-specific collection naming rules have been applied.
/// </summary>
// TODO: Make protected
protected abstract string CollectionNameBase { get; }
/// <summary>
/// The actual name of the test collection, after any provider-specific collection naming rules have been applied.
/// </summary>
public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
protected virtual string DistanceFunction => this.TestStore.DefaultDistanceFunction;
protected virtual string IndexKind => this.TestStore.DefaultIndexKind;
protected virtual VectorStoreCollection<TKey, TRecord> GetCollection()
=> this.TestStore.CreateCollection<TKey, TRecord>(this.CollectionName, this.CreateRecordDefinition());
public override async Task InitializeAsync()
{
await base.InitializeAsync();
this.Collection = this.GetCollection();
if (await this.Collection.CollectionExistsAsync())
{
await this.Collection.EnsureCollectionDeletedAsync();
}
await this.Collection.EnsureCollectionExistsAsync();
await this.SeedAsync();
}
public virtual VectorStoreCollection<TKey, TRecord> Collection { get; private set; } = null!;
public List<TRecord> TestData => this._testData ??= this.BuildTestData();
protected virtual async Task SeedAsync()
{
if (this.TestData.Count > 0)
{
await this.Collection.UpsertAsync(this.TestData);
await this.WaitForDataAsync();
}
}
protected virtual Task WaitForDataAsync()
=> this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count);
}
@@ -0,0 +1,42 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using Xunit;
namespace VectorData.ConformanceTests.Support;
public abstract class VectorStoreFixture : IAsyncLifetime
{
private int _nextKeyValue = 1;
public abstract TestStore TestStore { get; }
public virtual VectorStore VectorStore => this.TestStore.DefaultVectorStore;
public virtual string DefaultDistanceFunction => this.TestStore.DefaultDistanceFunction;
public virtual string DefaultIndexKind => this.TestStore.DefaultIndexKind;
public virtual Task InitializeAsync()
=> this.TestStore.ReferenceCountingStartAsync();
public virtual Task DisposeAsync()
=> this.TestStore.ReferenceCountingStopAsync();
public virtual TKey GenerateNextKey<TKey>()
=> this.TestStore.GenerateKey<TKey>(Interlocked.Increment(ref this._nextKeyValue));
/// <summary>
/// Creates a collection for the given name and definition.
/// Delegates to <see cref="TestStore.CreateCollection{TKey, TRecord}"/> which can be overridden for provider-specific options.
/// </summary>
public virtual VectorStoreCollection<TKey, TRecord> CreateCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition definition)
where TKey : notnull
where TRecord : class
=> this.TestStore.CreateCollection<TKey, TRecord>(name, definition);
/// <summary>
/// Creates a dynamic collection for the given name and definition.
/// Delegates to <see cref="TestStore.CreateDynamicCollection"/> which can be overridden for provider-specific options.
/// </summary>
public virtual VectorStoreCollection<object, Dictionary<string, object?>> CreateDynamicCollection(string name, VectorStoreCollectionDefinition definition)
=> this.TestStore.CreateDynamicCollection(name, definition);
}
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Reflection;
using System.Text.RegularExpressions;
using Xunit;
namespace VectorData.ConformanceTests;
/// <summary>
/// A test that ensures that all base test suites are implemented (or explicitly ignored) in provider implementations.
/// Used to make sure that test coverage is complete.
/// </summary>
public abstract class TestSuiteImplementationTests
{
protected virtual ICollection<Type> IgnoredTestBases { get; } = [];
[Fact]
public virtual void All_test_bases_must_be_implemented()
{
var concreteTests
= this.GetType().Assembly.GetTypes()
.Where(c => c.BaseType != typeof(object) && !c.IsAbstract && (c.IsPublic || c.IsNestedPublic))
.ToList();
var nonImplementedBases
= this.GetBaseTestClasses()
.Where(t => !this.IgnoredTestBases.Contains(t) && !concreteTests.Any(c => Implements(c, t)))
.Select(t => t.FullName)
.ToList();
Assert.False(
nonImplementedBases.Count > 0,
"\r\n-- Missing derived classes for --\r\n" + string.Join(Environment.NewLine, nonImplementedBases));
}
// Filter for abstract base types which end with Tests and possibly generic arity (e.g. FooTests`2)
protected virtual IEnumerable<Type> GetBaseTestClasses()
=> typeof(TestSuiteImplementationTests).Assembly.ExportedTypes
.Where(t => Regex.IsMatch(t.Name, """Tests(`\d+)?$""") && t.IsAbstract && !t.IsSealed && !t.IsInterface);
private static bool Implements(Type type, Type interfaceOrBaseType)
=> (type.IsPublic || type.IsNestedPublic) && interfaceOrBaseType.IsGenericTypeDefinition
? GetGenericTypeImplementations(type, interfaceOrBaseType).Any()
: interfaceOrBaseType.IsAssignableFrom(type);
private static IEnumerable<Type> GetGenericTypeImplementations(Type type, Type interfaceOrBaseType)
{
var typeInfo = type.GetTypeInfo();
if (!typeInfo.IsGenericTypeDefinition)
{
var baseTypes = interfaceOrBaseType.IsInterface
? typeInfo.ImplementedInterfaces
: GetBaseTypes(type);
foreach (var baseType in baseTypes)
{
if (baseType.IsGenericType
&& baseType.GetGenericTypeDefinition() == interfaceOrBaseType)
{
yield return baseType;
}
}
if (type.IsGenericType
&& type.GetGenericTypeDefinition() == interfaceOrBaseType)
{
yield return type;
}
}
}
private static IEnumerable<Type> GetBaseTypes(Type type)
{
var t = type.BaseType;
while (t != null)
{
yield return t;
t = t.BaseType;
}
}
}
@@ -0,0 +1,602 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Linq.Expressions;
using System.Reflection;
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.TypeTests;
public abstract class DataTypeTests<TKey, TRecord>(DataTypeTests<TKey, TRecord>.Fixture fixture) : DataTypeTests<TKey>()
where TKey : notnull
where TRecord : DataTypeTests<TKey>.RecordBase, new()
{
// Note: nullable value types are tested automatically within TestTypeStructAsync
[ConditionalFact]
public virtual Task Byte()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(byte))
? Task.CompletedTask
: this.Test<byte>("Byte", 8, 9);
[ConditionalFact]
public virtual Task Short()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(short))
? Task.CompletedTask
: this.Test<short>("Short", 8, 9);
[ConditionalFact]
public virtual Task Int()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(int))
? Task.CompletedTask
: this.Test<int>("Int", 8, 9);
[ConditionalFact]
public virtual Task Long()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(long))
? Task.CompletedTask
: this.Test<long>("Long", 8L, 9L);
[ConditionalFact]
public virtual Task Float()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(float))
? Task.CompletedTask
: this.Test<float>("Float", 8.5f, 9.5f);
[ConditionalFact]
public virtual Task Double()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(double))
? Task.CompletedTask
: this.Test<double>("Double", 8.5d, 9.5d);
[ConditionalFact]
public virtual Task Decimal()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(decimal))
? Task.CompletedTask
: this.Test<decimal>("Decimal", 8.5m, 9.5m);
[ConditionalFact]
public virtual Task String()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(string))
? Task.CompletedTask
: this.Test<string>("String", "foo", "bar");
[ConditionalFact]
public virtual Task Bool()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(bool))
? Task.CompletedTask
: this.Test<bool>("Bool", true, false);
[ConditionalFact]
public virtual Task Guid()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(Guid))
? Task.CompletedTask
: this.Test<Guid>(
"Guid",
new Guid("603840bf-cf91-4521-8b8e-8b6a2e75910a"),
new Guid("e9a97807-8cf0-4741-8ce3-82df676ca0f0"));
[ConditionalFact]
public virtual Task DateTime()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(DateTime))
? Task.CompletedTask
: this.Test<DateTime>(
"DateTime",
new DateTime(2020, 1, 1, 12, 30, 45),
new DateTime(2021, 2, 3, 13, 40, 55),
instantiationExpression: () => new DateTime(2020, 1, 1, 12, 30, 45));
[ConditionalFact]
public virtual Task DateTimeOffset()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(DateTimeOffset))
? Task.CompletedTask
: this.Test<DateTimeOffset>(
"DateTimeOffset",
new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(2)),
new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.FromHours(3)),
instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(2)));
[ConditionalFact]
public virtual Task DateOnly()
{
#if NET
return fixture.UnsupportedDefaultTypes.Contains(typeof(DateOnly))
? Task.CompletedTask
: this.Test<DateOnly>(
"DateOnly",
new DateOnly(2020, 1, 1),
new DateOnly(2021, 2, 3));
#else
return Task.CompletedTask;
#endif
}
[ConditionalFact]
public virtual Task TimeOnly()
{
#if NET
return fixture.UnsupportedDefaultTypes.Contains(typeof(TimeOnly))
? Task.CompletedTask
: this.Test<TimeOnly>(
"TimeOnly",
new TimeOnly(12, 30, 45),
new TimeOnly(13, 40, 55));
#else
return Task.CompletedTask;
#endif
}
[ConditionalFact]
public virtual Task String_array()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(string[]))
? Task.CompletedTask
: this.Test<string[]>(
"StringArray",
["foo", "bar"],
["foo", "baz"]);
[ConditionalFact]
public virtual Task Nullable_value_type()
=> fixture.UnsupportedDefaultTypes.Contains(typeof(int?))
? Task.CompletedTask
: this.Test<int?>("NullableInt", 8, 9);
protected virtual async Task Test<TTestType>(
string propertyName,
TTestType mainValue,
TTestType otherValue,
bool isFilterable = true,
Action<TTestType, TTestType>? comparisonAction = null,
Expression<Func<TTestType>>? instantiationExpression = null)
{
if (propertyName is "Key" or "Vector")
{
throw new ArgumentException($"The property name '{propertyName}' is reserved and cannot be used for testing.", nameof(propertyName));
}
var property = typeof(TRecord).GetProperty(propertyName)
?? throw new ArgumentException($"The type '{typeof(TRecord).Name}' does not have a property named '{propertyName}'.", nameof(propertyName));
comparisonAction ??= (a, b) => Assert.Equal(a, b);
var instantiationExpressionBody = instantiationExpression is null
? Expression.Constant(mainValue, typeof(TTestType))
: instantiationExpression.Body;
await fixture.Collection.DeleteAsync([fixture.MainRecordKey, fixture.OtherRecordKey, fixture.NullRecordKey]);
await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: 0);
// Step 1: Insert data
await this.InsertData(property, mainValue, otherValue);
// Step 2: Read the values back via GetAsync
TRecord result = await fixture.Collection.GetAsync(fixture.MainRecordKey) ?? throw new InvalidOperationException($"Record with key '{fixture.MainRecordKey}' was not found.");
comparisonAction(mainValue, (TTestType)property.GetValue(result)!);
// Step 3: Exercise filtering by the value, using a constant in the filter expression
if (isFilterable)
{
await this.TestFiltering(fixture.Collection, property, mainValue, comparisonAction, instantiationExpressionBody);
}
///////////////////////
// Test dynamic mapping
///////////////////////
if (fixture.RecreateCollection)
{
await fixture.Collection.EnsureCollectionDeletedAsync();
}
else
{
await fixture.Collection.DeleteAsync([fixture.MainRecordKey, fixture.OtherRecordKey, fixture.NullRecordKey]);
await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: 0);
}
var dynamicCollection = fixture.CreateDynamicCollection(fixture.CollectionName, fixture.CreateRecordDefinition());
if (fixture.RecreateCollection)
{
await dynamicCollection.EnsureCollectionExistsAsync();
}
// Step 1: Insert data
await this.InsertDynamicData(dynamicCollection, propertyName, mainValue, otherValue);
// Step 2: Read the values back via GetAsync
var dynamicResult = await dynamicCollection.GetAsync(fixture.MainRecordKey) ?? throw new InvalidOperationException($"Record with key '{fixture.MainRecordKey}' was not found.");
comparisonAction(mainValue, (TTestType)dynamicResult[propertyName]!);
// Step 3: Exercise dynamic filtering by the value, using a constant in the filter expression
if (isFilterable)
{
await this.TestDynamicFiltering(dynamicCollection, propertyName, mainValue, comparisonAction, instantiationExpressionBody);
}
}
private async Task InsertData<TTestType>(PropertyInfo property, TTestType mainValue, TTestType otherValue)
{
// Note that all records have the same vector
var mainRecord = GenerateEmptyRecord();
mainRecord.Key = fixture.MainRecordKey;
mainRecord.Vector = fixture.Vector;
property.SetValue(mainRecord, mainValue);
var otherRecord = GenerateEmptyRecord();
otherRecord.Key = fixture.OtherRecordKey;
otherRecord.Vector = fixture.Vector;
property.SetValue(otherRecord, otherValue);
List<TRecord> testData = [mainRecord, otherRecord];
if (default(TTestType) == null && fixture.IsNullSupported && IsPropertyNullable(property))
{
var nullRecord = GenerateEmptyRecord();
nullRecord.Key = fixture.NullRecordKey;
nullRecord.Vector = fixture.Vector;
property.SetValue(nullRecord, null);
testData.Add(nullRecord);
}
await fixture.Collection.UpsertAsync(testData);
await fixture.TestStore.WaitForDataAsync(fixture.Collection, recordCount: testData.Count);
TRecord GenerateEmptyRecord()
{
var record = new TRecord();
foreach (var property in fixture.CreateRecordDefinition().Properties)
{
var propertyInfo = typeof(TRecord).GetProperty(property.Name)
?? throw new InvalidOperationException($"Property '{property.Name}' not found on record type '{typeof(TRecord).Name}'.");
propertyInfo.SetValue(record, this.GenerateEmptyProperty(property));
}
return record;
}
}
private async Task InsertDynamicData<TTestType>(
VectorStoreCollection<object, Dictionary<string, object?>> dynamicCollection,
string propertyName,
TTestType mainValue,
TTestType otherValue)
{
// Note that all records have the same vector
var mainRecord = GenerateEmptyRecord();
mainRecord[nameof(RecordBase.Key)] = fixture.MainRecordKey;
mainRecord[nameof(RecordBase.Vector)] = fixture.Vector;
mainRecord[propertyName] = mainValue;
var otherRecord = GenerateEmptyRecord();
otherRecord[nameof(RecordBase.Key)] = fixture.OtherRecordKey;
otherRecord[nameof(RecordBase.Vector)] = fixture.Vector;
otherRecord[propertyName] = otherValue;
List<Dictionary<string, object?>> testData = [mainRecord, otherRecord];
var pocoProperty = typeof(TRecord).GetProperty(propertyName);
if (default(TTestType) == null && fixture.IsNullSupported && (pocoProperty is null || IsPropertyNullable(pocoProperty)))
{
var nullRecord = GenerateEmptyRecord();
nullRecord[nameof(RecordBase.Key)] = fixture.NullRecordKey;
nullRecord[nameof(RecordBase.Vector)] = fixture.Vector;
nullRecord[propertyName] = null;
testData.Add(nullRecord);
}
await dynamicCollection.UpsertAsync(testData);
await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: testData.Count);
Dictionary<string, object?> GenerateEmptyRecord()
{
var record = new Dictionary<string, object?>();
foreach (var property in fixture.CreateRecordDefinition().Properties)
{
record[property.Name] = this.GenerateEmptyProperty(property);
}
return record;
}
}
/// <summary>
/// Checks whether a property is nullable, taking into account NRT annotations on .NET 6+.
/// </summary>
private static bool IsPropertyNullable(PropertyInfo property)
{
if (property.PropertyType.IsValueType)
{
return Nullable.GetUnderlyingType(property.PropertyType) is not null;
}
#if NET
return new NullabilityInfoContext().Create(property).ReadState != NullabilityState.NotNull;
#else
return true; // Without NRT support, assume reference types are nullable
#endif
}
protected virtual object? GenerateEmptyProperty(VectorStoreProperty property)
=> property.Type switch
{
null => throw new InvalidOperationException($"Property '{property.Name}' has no type defined."),
// For value types, we create an instance with the default value.
// This is necessary for relational providers where non-nullable columns are created.
var t when t.IsValueType => Activator.CreateInstance(t),
// In some cases (Azure AI Search), array fields must be non-null
var t when t.IsArray => Array.CreateInstance(t.GetElementType()!, 0),
_ => null
};
private async Task TestFiltering<TTestType>(
VectorStoreCollection<TKey, TRecord> collection,
PropertyInfo property,
TTestType mainValue,
Action<TTestType, TTestType> comparisonAction,
Expression instantiationExpression)
{
// Note: we need to manually build the expression tree since the equality operator can't be used over
// unconstrained generic types.
var lambdaParameter = Expression.Parameter(typeof(TRecord), "r");
var filter = Expression.Lambda<Func<TRecord, bool>>(
Expression.Equal(
Expression.Property(lambdaParameter, property),
instantiationExpression),
lambdaParameter);
// Some databases (Mongo) update the filter index asynchronously, so we wait until the record appears under the filter,
// and then do the main search to make sure only the main record is returned.
await fixture.TestStore.WaitForDataAsync(collection, filter: filter, recordCount: 1);
var result = (await collection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record;
Assert.Equal(fixture.MainRecordKey, result.Key);
comparisonAction(mainValue, (TTestType)property.GetValue(result)!);
// Exercise filtering by a null value
if (default(TTestType) == null && fixture.IsNullFilteringSupported && IsPropertyNullable(property))
{
lambdaParameter = Expression.Parameter(typeof(TRecord), "r");
filter = Expression.Lambda<Func<TRecord, bool>>(
Expression.Equal(
Expression.Property(lambdaParameter, property),
Expression.Constant(null, typeof(TTestType))),
lambdaParameter);
result = (await collection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record;
Assert.Equal(fixture.NullRecordKey, result.Key);
}
}
private async Task TestDynamicFiltering<TTestType>(
VectorStoreCollection<object, Dictionary<string, object?>> dynamicCollection,
string propertyName,
TTestType mainValue,
Action<TTestType, TTestType> comparisonAction,
Expression instantiationExpression)
{
// Note: we need to manually build the expression tree since we want the property name to be a constant
var lambdaParameter = Expression.Parameter(typeof(Dictionary<string, object>), "r");
var filter = Expression.Lambda<Func<Dictionary<string, object?>, bool>>(
Expression.Equal(
Expression.Convert(
Expression.Call(lambdaParameter, DynamicDictionaryIndexer, Expression.Constant(propertyName)),
typeof(TTestType)),
instantiationExpression),
lambdaParameter);
// Some databases (Mongo) update the filter index asynchronously, so we wait until the record appears under the filter,
// and then do the main search to make sure only the main record is returned.
await fixture.TestStore.WaitForDataAsync(dynamicCollection, filter: filter, recordCount: 1);
var result = (await dynamicCollection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record;
Assert.Equal(fixture.MainRecordKey, result[nameof(RecordBase.Key)]);
comparisonAction(mainValue, (TTestType)result[propertyName]!);
// Exercise filtering by a null value
var pocoProperty = typeof(TRecord).GetProperty(propertyName);
if (default(TTestType) == null && fixture.IsNullFilteringSupported && (pocoProperty is null || IsPropertyNullable(pocoProperty)))
{
lambdaParameter = Expression.Parameter(typeof(Dictionary<string, object?>), "r");
filter = Expression.Lambda<Func<Dictionary<string, object?>, bool>>(
Expression.Equal(
Expression.Convert(
Expression.Call(lambdaParameter, DynamicDictionaryIndexer, Expression.Constant(propertyName)),
typeof(TTestType)),
Expression.Constant(null, typeof(TTestType))),
lambdaParameter);
result = (await dynamicCollection.SearchAsync(fixture.Vector, top: 100, new() { Filter = filter }).SingleAsync()).Record;
Assert.Equal(fixture.NullRecordKey, result[nameof(RecordBase.Key)]);
}
}
private static readonly MethodInfo DynamicDictionaryIndexer = typeof(Dictionary<string, object?>).GetMethod("get_Item")!;
public abstract class Fixture : VectorStoreCollectionFixture<TKey, TRecord>
{
protected override string CollectionNameBase => nameof(DataTypeTests<int>);
public virtual bool IsNullSupported => true;
public virtual bool IsNullFilteringSupported => true;
public virtual Type[] UnsupportedDefaultTypes { get; } = [];
public virtual TKey MainRecordKey { get; protected set; } = default!;
public virtual TKey OtherRecordKey { get; protected set; } = default!;
public virtual TKey NullRecordKey { get; protected set; } = default!;
public virtual float[] Vector { get; } = [1, 2, 3];
private readonly IList<VectorStoreDataProperty> _defaultDataProperties = null!;
/// <summary>
/// Whether the recreate the collection while testing, as opposed to deleting the records.
/// Necessary for InMemory, where the .NET mapped on the collection cannot be changed.
/// </summary>
public virtual bool RecreateCollection => false;
#pragma warning disable CA2214 // Do not call overridable methods in constructors
protected Fixture()
{
this._defaultDataProperties = this.GetDataProperties();
}
#pragma warning restore CA2214
public override async Task InitializeAsync()
{
await base.InitializeAsync();
this.MainRecordKey = this.GenerateNextKey<TKey>();
this.OtherRecordKey = this.GenerateNextKey<TKey>();
this.NullRecordKey = this.GenerateNextKey<TKey>();
}
public override VectorStoreCollectionDefinition CreateRecordDefinition()
=> new()
{
Properties =
[
new VectorStoreKeyProperty(nameof(RecordBase.Key), typeof(TKey)),
new VectorStoreVectorProperty(nameof(RecordBase.Vector), typeof(float[]), 3)
{
DistanceFunction = this.DistanceFunction,
IndexKind = this.IndexKind
},
.. this._defaultDataProperties
]
};
public virtual IList<VectorStoreDataProperty> GetDataProperties()
{
var properties = new List<VectorStoreDataProperty>();
if (!this.UnsupportedDefaultTypes.Contains(typeof(byte)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Byte), typeof(byte)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(short)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Short), typeof(short)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(int)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Int), typeof(int)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(long)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Long), typeof(long)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(float)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Float), typeof(float)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(double)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Double), typeof(double)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(decimal)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Decimal), typeof(decimal)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(string)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.String), typeof(string)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(bool)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Bool), typeof(bool)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(Guid)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.Guid), typeof(Guid)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(DateTime)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateTime), typeof(DateTime)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(DateTimeOffset)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateTimeOffset), typeof(DateTimeOffset)) { IsIndexed = true });
}
#if NET
if (!this.UnsupportedDefaultTypes.Contains(typeof(DateOnly)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.DateOnly), typeof(DateOnly)) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(TimeOnly)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.TimeOnly), typeof(TimeOnly)) { IsIndexed = true });
}
#endif
if (!this.UnsupportedDefaultTypes.Contains(typeof(string[])))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.StringArray), typeof(string[])) { IsIndexed = true });
}
if (!this.UnsupportedDefaultTypes.Contains(typeof(int?)))
{
properties.Add(new VectorStoreDataProperty(nameof(DefaultRecord.NullableInt), typeof(int?)) { IsIndexed = true });
}
return properties;
}
}
}
// We have this base class so the Record type can be referenced in subtypes (the main TypeTests class
// is generic over the record type as well).
public abstract class DataTypeTests<TKey>()
where TKey : notnull
{
public class RecordBase : TestRecord<TKey>
{
public float[] Vector { get; set; } = default!;
}
public class DefaultRecord : RecordBase
{
public byte Byte { get; set; }
public short Short { get; set; }
public int Int { get; set; }
public long Long { get; set; }
public float Float { get; set; }
public double Double { get; set; }
public decimal Decimal { get; set; }
public string? String { get; set; }
public bool Bool { get; set; }
public Guid Guid { get; set; }
public DateTime DateTime { get; set; }
public DateTimeOffset DateTimeOffset { get; set; }
#if NET
public DateOnly DateOnly { get; set; }
public TimeOnly TimeOnly { get; set; }
#endif
public string[] StringArray { get; set; } = null!;
public int? NullableInt { get; set; }
}
}
@@ -0,0 +1,252 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
#pragma warning disable CA2000 // Dispose objects before losing scope
namespace VectorData.ConformanceTests.TypeTests;
/// <summary>
/// Tests that the various embedding types natively supported by the provider (<c>ReadOnlyMemory&lt;float&gt;</c>, <c>ReadOnlyMemory&lt;Half&gt;</c>...) work correctly.
/// </summary>
public abstract class EmbeddingTypeTests<TKey>(EmbeddingTypeTests<TKey>.Fixture fixture)
where TKey : notnull
{
[ConditionalFact]
public virtual Task ReadOnlyMemory_of_float()
=> this.Test<ReadOnlyMemory<float>>(
new ReadOnlyMemory<float>([1, 2, 3]),
new ReadOnlyMemoryEmbeddingGenerator<float>([1, 2, 3]),
vectorEqualityAsserter: (e, a) => Assert.Equal(e.ToArray(), a.ToArray()));
[ConditionalFact]
public virtual Task Embedding_of_float()
=> this.Test<Embedding<float>>(
new Embedding<float>(new ReadOnlyMemory<float>([1, 2, 3])),
new ReadOnlyMemoryEmbeddingGenerator<float>([1, 2, 3]),
vectorEqualityAsserter: (e, a) => Assert.Equal(e.Vector.ToArray(), a.Vector.ToArray()));
[ConditionalFact]
public virtual Task Array_of_float()
=> this.Test<float[]>(
[1, 2, 3],
new ReadOnlyMemoryEmbeddingGenerator<float>([1, 2, 3]));
protected virtual async Task Test<TVector>(
TVector value,
IEmbeddingGenerator? embeddingGenerator = null,
Action<TVector, TVector>? vectorEqualityAsserter = null,
string? distanceFunction = null,
int dimensions = 3)
where TVector : notnull
{
vectorEqualityAsserter ??= (e, a) => Assert.Equal(e, a);
await fixture.VectorStore.EnsureCollectionDeletedAsync(fixture.CollectionName);
var collection = fixture.CreateCollection<TKey, Record<TVector>>(fixture.CollectionName, fixture.CreateRecordDefinition<TVector>(embeddingGenerator: null, distanceFunction, dimensions));
await collection.EnsureCollectionExistsAsync();
var key = fixture.GenerateNextKey<TKey>();
var record = new Record<TVector>
{
Key = key,
Vector = value,
Int = 42
};
await collection.UpsertAsync(record);
await fixture.TestStore.WaitForDataAsync(collection, recordCount: 1, filter: r => r.Int == 42, dummyVector: value);
var result1 = await collection.GetAsync(key, new() { IncludeVectors = true });
Assert.Equal(42, result1!.Int);
if (fixture.TestStore.VectorsComparable)
{
vectorEqualityAsserter(value, result1.Vector);
}
// Note that some databases leak indexing information across deletion/recreation of the same collection name (e.g. Pinecone) - so we also
// filter by Int here.
var result2 = await collection.SearchAsync(value, top: 1, new() { IncludeVectors = true, Filter = r => r.Int == 42 }).SingleAsync();
Assert.Equal(42, result2.Record.Int);
if (fixture.TestStore.VectorsComparable)
{
vectorEqualityAsserter(value, result2.Record.Vector);
}
///////////////////////
// Test dynamic mapping
///////////////////////
if (fixture.RecreateCollection)
{
await collection.EnsureCollectionDeletedAsync();
}
else
{
await collection.DeleteAsync(key);
}
var dynamicCollection = fixture.CreateDynamicCollection(fixture.CollectionName, fixture.CreateRecordDefinition<TVector>(embeddingGenerator, distanceFunction, dimensions));
if (fixture.RecreateCollection)
{
await dynamicCollection.EnsureCollectionExistsAsync();
}
key = fixture.GenerateNextKey<TKey>();
var dynamicRecord = new Dictionary<string, object?>
{
["Key"] = key,
["Vector"] = value,
["Int"] = 43
};
await dynamicCollection.UpsertAsync(dynamicRecord);
await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: 1, filter: r => (int)r["Int"]! == 43, dummyVector: value);
var dynamicResult1 = await dynamicCollection.GetAsync(key, new() { IncludeVectors = true });
Assert.Equal(43, dynamicResult1!["Int"]);
if (fixture.TestStore.VectorsComparable)
{
vectorEqualityAsserter(value, (TVector)dynamicResult1["Vector"]!);
}
// Note that some databases leak indexing information across deletion/recreation of the same collection name (e.g. Pinecone) - so we also
// filter by Int here.
var dynamicResult2 = await dynamicCollection.SearchAsync(value, top: 1, new() { IncludeVectors = true, Filter = r => (int)r["Int"]! == 43 }).SingleAsync();
Assert.Equal(43, dynamicResult2.Record["Int"]);
if (fixture.TestStore.VectorsComparable)
{
vectorEqualityAsserter(value, (TVector)dynamicResult2.Record["Vector"]!);
}
////////////////////////////
// Test embedding generation
////////////////////////////
if (embeddingGenerator is not null)
{
if (fixture.RecreateCollection)
{
await collection.EnsureCollectionDeletedAsync();
}
else
{
await collection.DeleteAsync(key);
}
var collection2 = fixture.CreateCollection<TKey, RecordWithString>(fixture.CollectionName, fixture.CreateRecordDefinition<string>(embeddingGenerator, distanceFunction, dimensions));
if (fixture.RecreateCollection)
{
await collection2.EnsureCollectionExistsAsync();
}
key = fixture.GenerateNextKey<TKey>();
var record2 = new RecordWithString
{
Key = key,
Vector = "does not matter",
Int = 44
};
await collection2.UpsertAsync(record2);
await fixture.TestStore.WaitForDataAsync(collection2, recordCount: 1, filter: r => r.Int == 44, dummyVector: value);
var result3 = await collection2.GetAsync(key);
Assert.Equal(44, result3!.Int);
if (fixture.AssertNoVectorsLoadedWithEmbeddingGeneration)
{
Assert.Null(result3.Vector);
}
var result4 = await collection2.SearchAsync("does not matter", top: 1, new() { Filter = r => r.Int == 44 }).SingleAsync();
Assert.Equal(44, result4.Record.Int);
if (fixture.AssertNoVectorsLoadedWithEmbeddingGeneration)
{
Assert.Null(result4.Record.Vector);
}
}
}
protected sealed class ReadOnlyMemoryEmbeddingGenerator<T>(T[] data) : IEmbeddingGenerator<string, Embedding<T>>
{
public Task<GeneratedEmbeddings<Embedding<T>>> GenerateAsync(
IEnumerable<string> values,
EmbeddingGenerationOptions? options = null,
CancellationToken cancellationToken = default)
=> Task.FromResult(new GeneratedEmbeddings<Embedding<T>>([new(data)]));
public object? GetService(Type serviceType, object? serviceKey = null) => null;
public void Dispose() { }
}
protected sealed class BinaryEmbeddingGenerator(BitArray data) : IEmbeddingGenerator<string, BinaryEmbedding>
{
public Task<GeneratedEmbeddings<BinaryEmbedding>> GenerateAsync(
IEnumerable<string> values,
EmbeddingGenerationOptions? options = null,
CancellationToken cancellationToken = default)
=> Task.FromResult(new GeneratedEmbeddings<BinaryEmbedding>([new(data)]));
public object? GetService(Type serviceType, object? serviceKey = null) => null;
public void Dispose() { }
}
public class RecordWithString
{
public TKey Key { get; set; }
public string Vector { get; set; }
public int Int { get; set; }
}
public class Record<TVector>
{
public TKey Key { get; set; }
public TVector Vector { get; set; }
public int Int { get; set; }
}
public abstract class Fixture : VectorStoreFixture
{
protected virtual string CollectionNameBase => nameof(EmbeddingTypeTests<>);
public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
public virtual VectorStoreCollectionDefinition CreateRecordDefinition<TVectorProperty>(IEmbeddingGenerator? embeddingGenerator, string? distanceFunction, int dimensions)
=> new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(TKey)),
new VectorStoreVectorProperty("Vector", typeof(TVectorProperty), dimensions)
{
DistanceFunction = distanceFunction ?? this.DefaultDistanceFunction,
IndexKind = this.DefaultIndexKind
},
new VectorStoreDataProperty("Int", typeof(int)) { IsIndexed = true }
],
EmbeddingGenerator = embeddingGenerator
};
/// <summary>
/// Whether the recreate the collection while testing, as opposed to deleting the records.
/// Necessary for InMemory, where the .NET mapped on the collection cannot be changed.
/// </summary>
public virtual bool RecreateCollection => false;
/// <summary>
/// Whether to assert that no vectors were loaded when embedding generation is used.
/// Necessary for InMemory which returns the same object which was inserted, and therefore contains
/// the original input value.
/// </summary>
public virtual bool AssertNoVectorsLoadedWithEmbeddingGeneration => true;
}
}
@@ -0,0 +1,254 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
using VectorData.ConformanceTests.Support;
using VectorData.ConformanceTests.Xunit;
using Xunit;
namespace VectorData.ConformanceTests.TypeTests;
public abstract class KeyTypeTests(KeyTypeTests.Fixture fixture)
{
// All MEVD providers are expected to support Guid keys (possibly by storing them as strings), including
// auto-generation.
// This allows upper layers such as Microsoft.Extensions.DataIngestion to use Guid keys consistently.
[ConditionalFact]
public virtual Task Guid()
=> this.Test<Guid>(
new Guid("603840bf-cf91-4521-8b8e-8b6a2e75910a"),
supportsAutoGeneration: true);
/// <summary>
/// Verifies that creating a collection with a TKey that doesn't match the key property type on the model throws.
/// </summary>
[ConditionalFact]
public virtual void MismatchedKeyTypeThrows()
{
// The definition says the key property is string (matching Record<string>.Key),
// but TKey is Guid - this mismatch should be detected during model building.
Assert.Throws<InvalidOperationException>(() =>
fixture.TestStore.CreateCollection<Guid, Record<string>>(
fixture.CollectionName, fixture.CreateRecordDefinition<string>(withAutoGeneration: false)));
}
protected virtual Task Test<TKey>(TKey key, bool supportsAutoGeneration = false)
where TKey : struct
=> this.Test<TKey>(key, default!, supportsAutoGeneration: supportsAutoGeneration);
// Note that we do not currently support testing auto generation for reference types, since
// no such case currently exists in a known provider. As a result we require a second key
// value.
protected virtual Task Test<TKey>(TKey key1, TKey key2)
where TKey : class
=> this.Test<TKey>(key1, key2, supportsAutoGeneration: false);
protected virtual async Task Test<TKey>(
TKey key1,
TKey key2,
bool supportsAutoGeneration = false)
where TKey : notnull
{
Assert.NotEqual(key1, key2);
using var collection = fixture.CreateCollection<TKey>(withAutoGeneration: false);
{
await collection.EnsureCollectionDeletedAsync();
await collection.EnsureCollectionExistsAsync();
var record = new Record<TKey>
{
Key = key1,
Int = 8,
Vector = new ReadOnlyMemory<float>([1, 2, 3])
};
var nextRecord = new Record<TKey>
{
Key = key2,
Int = 9,
Vector = new ReadOnlyMemory<float>([3, 2, 1])
};
await collection.UpsertAsync(record);
// Exercise multi-record plus updating existing record
await collection.UpsertAsync([record, nextRecord]);
await fixture.TestStore.WaitForDataAsync(collection, recordCount: 2);
// Single record get
var result = await collection.GetAsync(key1);
Assert.NotNull(result);
Assert.Equal(key1, result.Key);
Assert.Equal(8, result.Int);
// Multiple record get
// Also ensures that the second record - with the default key value - got properly inserted and did not trigger auto-generation
// (as we haven't configured it).
var results = await collection.GetAsync([key1, key2]).ToListAsync();
Assert.Equal(2, results.Count);
var firstRecord = Assert.Single(results, r => r.Key.Equals(key1));
Assert.Equal(8, firstRecord.Int);
var secondRecord = Assert.Single(results, r => r.Key.Equals(key2));
Assert.Equal(9, secondRecord.Int);
}
///////////////////////
// Test dynamic mapping
///////////////////////
await collection.DeleteAsync(key1);
await collection.DeleteAsync([key1, key2]);
await fixture.TestStore.WaitForDataAsync(collection, recordCount: 0);
using (var dynamicCollection = fixture.CreateDynamicCollection<TKey>(withAutoGeneration: false))
{
await dynamicCollection.EnsureCollectionExistsAsync();
var dynamicRecord = new Dictionary<string, object?>
{
[nameof(Record<TKey>.Key)] = key1,
[nameof(Record<TKey>.Int)] = 8,
[nameof(Record<TKey>.Vector)] = new ReadOnlyMemory<float>([1, 2, 3])
};
var nextDynamicRecord = new Dictionary<string, object?>
{
[nameof(Record<TKey>.Key)] = key2,
[nameof(Record<TKey>.Int)] = 9,
[nameof(Record<TKey>.Vector)] = new ReadOnlyMemory<float>([3, 2, 1])
};
await dynamicCollection.UpsertAsync(dynamicRecord);
// Exercise multi-record plus updating existing record
await dynamicCollection.UpsertAsync([dynamicRecord, nextDynamicRecord]);
await fixture.TestStore.WaitForDataAsync(dynamicCollection, recordCount: 2);
// Single record get
var dynamicResult = await dynamicCollection.GetAsync(key1);
Assert.NotNull(dynamicResult);
Assert.IsType<TKey>(dynamicResult[nameof(Record<TKey>.Key)]);
Assert.Equal(key1, (TKey)dynamicResult[nameof(Record<TKey>.Key)]!);
Assert.Equal(8, dynamicResult[nameof(Record<TKey>.Int)]);
// Multiple record get
// Also ensures that the second record - with the default key value - got properly inserted and did not trigger auto-generation
// (as we haven't configured it).
var dynamicResults = await dynamicCollection.GetAsync([key1, key2]).ToListAsync();
Assert.Equal(2, dynamicResults.Count);
var firstDynamicRecord = Assert.Single(dynamicResults, r => r[nameof(Record<TKey>.Key)]!.Equals(key1));
Assert.IsType<TKey>(firstDynamicRecord[nameof(Record<TKey>.Key)]);
Assert.Equal(8, firstDynamicRecord[nameof(Record<TKey>.Int)]);
var secondDynamicRecord = Assert.Single(dynamicResults, r => r[nameof(Record<TKey>.Key)]!.Equals(key2));
Assert.IsType<TKey>(secondDynamicRecord[nameof(Record<TKey>.Key)]);
Assert.Equal(9, secondDynamicRecord[nameof(Record<TKey>.Int)]);
}
if (supportsAutoGeneration)
{
// Above we tested with a collection where auto-generation isn't enabled - including with the default key value,
// which would have triggered auto-generation if it was enabled.
// Now, drop and recreate the collection with auto-generation enabled, and test that it works.
await collection.EnsureCollectionDeletedAsync();
// Pass null to test the provider's default behavior, which should be to enable auto-generation.
using var collectionWithAutoGeneration = fixture.CreateCollection<TKey>(withAutoGeneration: null);
await collectionWithAutoGeneration.EnsureCollectionExistsAsync();
var record = new Record<TKey>
{
Key = key1,
Int = 8,
Vector = new ReadOnlyMemory<float>([1, 2, 3])
};
var recordWithDefaultValueKey1 = new Record<TKey>
{
Key = key2,
Int = 9,
Vector = new ReadOnlyMemory<float>([3, 2, 1])
};
var recordWithDefaultValueKey2 = new Record<TKey>
{
Key = key2,
Int = 10,
Vector = new ReadOnlyMemory<float>([3, 2, 1])
};
var recordWithDefaultValueKey3 = new Record<TKey>
{
Key = key2,
Int = 11,
Vector = new ReadOnlyMemory<float>([3, 2, 1])
};
// recordWithDefaultValueKey1 gets inserted alone, exercising single-record upsert with auto-generation.
await collectionWithAutoGeneration.UpsertAsync(recordWithDefaultValueKey1);
Assert.NotEqual(recordWithDefaultValueKey1.Key, key2);
var preUpdateGeneratedKey = recordWithDefaultValueKey1.Key;
recordWithDefaultValueKey1.Int = 99;
// recordWithDefaultValueKey1 gets upserted, exercising update instead of insert.
// recordWithDefaultValueKey2 and 3 get inserted, exercising multi-record upsert with auto-generation; we insert two records to make
// sure the correct key gets injected back into each record.
// Finally, record gets inserted with a non-generated key, to make sure auto-generation doesn't kick in for non-CLR-default keys.
await collectionWithAutoGeneration.UpsertAsync([recordWithDefaultValueKey1, recordWithDefaultValueKey2, recordWithDefaultValueKey3, record]);
await fixture.TestStore.WaitForDataAsync(collectionWithAutoGeneration, recordCount: 4);
Assert.Equal(recordWithDefaultValueKey1.Key, preUpdateGeneratedKey);
Assert.Equal(99, recordWithDefaultValueKey1.Int);
Assert.NotEqual(recordWithDefaultValueKey2.Key, key2);
Assert.NotEqual(recordWithDefaultValueKey3.Key, key2);
Assert.NotEqual(recordWithDefaultValueKey2.Key, recordWithDefaultValueKey1.Key!);
Assert.NotEqual(recordWithDefaultValueKey3.Key, recordWithDefaultValueKey1.Key!);
Assert.NotEqual(recordWithDefaultValueKey3.Key, recordWithDefaultValueKey2.Key!);
Assert.Equal(record.Key, key1);
var results = await collectionWithAutoGeneration.GetAsync([key1, recordWithDefaultValueKey1.Key, recordWithDefaultValueKey2.Key, recordWithDefaultValueKey3.Key]).ToListAsync();
Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey1.Key));
Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey2.Key));
Assert.Single(results, r => r.Key.Equals(recordWithDefaultValueKey3.Key));
Assert.Single(results, r => r.Key.Equals(key1));
}
else
{
// Auto-generation is not supported for this type; ensure that model validation throws.
Assert.Throws<NotSupportedException>(() => fixture.CreateCollection<TKey>(withAutoGeneration: true));
}
}
public abstract class Fixture : VectorStoreFixture
{
protected virtual string CollectionNameBase => nameof(KeyTypeTests);
public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
public virtual VectorStoreCollection<TKey, Record<TKey>> CreateCollection<TKey>(bool? withAutoGeneration)
where TKey : notnull
=> this.TestStore.CreateCollection<TKey, Record<TKey>>(this.CollectionName, this.CreateRecordDefinition<TKey>(withAutoGeneration));
public virtual VectorStoreCollection<object, Dictionary<string, object?>> CreateDynamicCollection<TKey>(bool withAutoGeneration)
where TKey : notnull
=> this.TestStore.CreateDynamicCollection(this.CollectionName, this.CreateRecordDefinition<TKey>(withAutoGeneration));
public virtual VectorStoreCollectionDefinition CreateRecordDefinition<TKey>(bool? withAutoGeneration)
where TKey : notnull
=> new()
{
Properties =
[
new VectorStoreKeyProperty("Key", typeof(TKey)) { IsAutoGenerated = withAutoGeneration },
new VectorStoreDataProperty("Int", typeof(int)),
new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), dimensions: 3)
{
DistanceFunction = this.DefaultDistanceFunction,
IndexKind = this.DefaultIndexKind
}
]
};
}
public class Record<TKey>
{
public TKey Key { get; set; } = default!;
public int Int { get; set; }
public ReadOnlyMemory<float> Vector { get; set; }
}
}
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net10.0;net472</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<RootNamespace>VectorData.ConformanceTests</RootNamespace>
<NoWarn>$(NoWarn);MEVD9000</NoWarn> <!-- Microsoft.Extensions.VectorData experimental user-facing APIs -->
<NoWarn>$(NoWarn);MEVD9001</NoWarn> <!-- Microsoft.Extensions.VectorData experimental provider-facing APIs -->
</PropertyGroup>
<ItemGroup>
<PackageReference Include="xunit" />
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="System.Linq.AsyncEnumerable" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net472' ">
<PackageReference Include="System.Memory.Data" />
</ItemGroup>
</Project>
@@ -0,0 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
[AttributeUsage(AttributeTargets.Method)]
[XunitTestCaseDiscoverer("VectorData.ConformanceTests.Xunit.ConditionalFactDiscoverer", "VectorData.ConformanceTests")]
public sealed class ConditionalFactAttribute : FactAttribute;
@@ -0,0 +1,23 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit.Abstractions;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
/// <summary>
/// Used dynamically from <see cref="ConditionalFactAttribute" />.
/// Make sure to update that class if you move this type.
/// </summary>
public class ConditionalFactDiscoverer(IMessageSink messageSink) : FactDiscoverer(messageSink)
{
protected override IXunitTestCase CreateTestCase(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute)
=> new ConditionalFactTestCase(
this.DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod);
}
@@ -0,0 +1,39 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit.Abstractions;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
public sealed class ConditionalFactTestCase : XunitTestCase
{
[Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
public ConditionalFactTestCase()
{
}
public ConditionalFactTestCase(
IMessageSink diagnosticMessageSink,
TestMethodDisplay defaultMethodDisplay,
TestMethodDisplayOptions defaultMethodDisplayOptions,
ITestMethod testMethod,
object[]? testMethodArguments = null)
: base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments)
{
}
public override async Task<RunSummary> RunAsync(
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
=> await XunitTestCaseExtensions.TrySkipAsync(this, messageBus)
? new RunSummary { Total = 1, Skipped = 1 }
: await base.RunAsync(
diagnosticMessageSink,
messageBus,
constructorArguments,
aggregator,
cancellationTokenSource);
}
@@ -0,0 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
[AttributeUsage(AttributeTargets.Method)]
[XunitTestCaseDiscoverer("VectorData.ConformanceTests.Xunit.ConditionalTheoryDiscoverer", "VectorData.ConformanceTests")]
public sealed class ConditionalTheoryAttribute : TheoryAttribute;
@@ -0,0 +1,39 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit.Abstractions;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
/// <summary>
/// Used dynamically from <see cref="ConditionalTheoryAttribute" />.
/// Make sure to update that class if you move this type.
/// </summary>
public class ConditionalTheoryDiscoverer(IMessageSink messageSink) : TheoryDiscoverer(messageSink)
{
protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo theoryAttribute)
{
yield return new ConditionalTheoryTestCase(
this.DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod);
}
protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo theoryAttribute,
object[] dataRow)
{
yield return new ConditionalFactTestCase(
this.DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
dataRow);
}
}
@@ -0,0 +1,38 @@
// Copyright (c) Microsoft. All rights reserved.
using Xunit.Abstractions;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
public sealed class ConditionalTheoryTestCase : XunitTheoryTestCase
{
[Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
public ConditionalTheoryTestCase()
{
}
public ConditionalTheoryTestCase(
IMessageSink diagnosticMessageSink,
TestMethodDisplay defaultMethodDisplay,
TestMethodDisplayOptions defaultMethodDisplayOptions,
ITestMethod testMethod)
: base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
{
}
public override async Task<RunSummary> RunAsync(
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
=> await XunitTestCaseExtensions.TrySkipAsync(this, messageBus)
? new RunSummary { Total = 1, Skipped = 1 }
: await base.RunAsync(
diagnosticMessageSink,
messageBus,
constructorArguments,
aggregator,
cancellationTokenSource);
}
@@ -0,0 +1,20 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VectorData.ConformanceTests.Xunit;
/// <summary>
/// Disable the tests in the decorated scope.
/// </summary>
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
public sealed class DisableTestsAttribute : Attribute, ITestCondition
{
public ValueTask<bool> IsMetAsync()
{
return new(false);
}
public string Skip { get; set; } = "Test disabled due to usage of DisableTestsAttribute";
public string SkipReason
=> this.Skip;
}
@@ -0,0 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VectorData.ConformanceTests.Xunit;
public interface ITestCondition
{
ValueTask<bool> IsMetAsync();
string SkipReason { get; }
}
@@ -0,0 +1,51 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Concurrent;
using Xunit.Abstractions;
using Xunit.Sdk;
namespace VectorData.ConformanceTests.Xunit;
public static class XunitTestCaseExtensions
{
private static readonly ConcurrentDictionary<string, List<IAttributeInfo>> s_typeAttributes = new();
private static readonly ConcurrentDictionary<string, List<IAttributeInfo>> s_assemblyAttributes = new();
public static async ValueTask<bool> TrySkipAsync(XunitTestCase testCase, IMessageBus messageBus)
{
var method = testCase.Method;
var type = testCase.TestMethod.TestClass.Class;
var assembly = type.Assembly;
var skipReasons = new List<string>();
var attributes =
s_assemblyAttributes.GetOrAdd(
assembly.Name,
a => assembly.GetCustomAttributes(typeof(ITestCondition)).ToList())
.Concat(
s_typeAttributes.GetOrAdd(
type.Name,
t => type.GetCustomAttributes(typeof(ITestCondition)).ToList()))
.Concat(method.GetCustomAttributes(typeof(ITestCondition)))
.OfType<ReflectionAttributeInfo>()
.Select(attributeInfo => (ITestCondition)attributeInfo.Attribute);
foreach (var attribute in attributes)
{
if (!await attribute.IsMetAsync())
{
skipReasons.Add(attribute.SkipReason);
}
}
if (skipReasons.Count > 0)
{
messageBus.QueueMessage(
new TestSkipped(new XunitTest(testCase, testCase.DisplayName), string.Join(Environment.NewLine, skipReasons)));
return true;
}
return false;
}
}