chore: import upstream snapshot with attribution

dotnet/test/VectorData/VectorData.ConformanceTests/Support/DynamicVectorStoreCollectionFixture.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VectorData.ConformanceTests.Support;
+
+public abstract class DynamicVectorStoreCollectionFixture<TKey> : VectorStoreCollectionFixtureBase<object, Dictionary<string, object?>>
+    where TKey : notnull
+{
+    protected abstract string KeyPropertyName { get; }
+
+    public virtual async Task ReseedAsync()
+    {
+        // TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830
+
+        const int BatchSize = 100;
+
+        List<TKey> keys = [];
+        do
+        {
+            await foreach (var record in this.Collection.GetAsync(r => true, top: BatchSize))
+            {
+                // TODO: We don't use batching delete because of https://github.com/microsoft/semantic-kernel/issues/13303
+                await this.Collection.DeleteAsync((TKey)record[this.KeyPropertyName]!);
+            }
+        } while (keys.Count == BatchSize);
+
+        await this.SeedAsync();
+    }
+}

dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestRecord.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+
+namespace VectorData.ConformanceTests.Support;
+
+public abstract class TestRecord<TKey>
+{
+    [VectorStoreKey]
+    public TKey Key { get; set; } = default!;
+}

dotnet/test/VectorData/VectorData.ConformanceTests/Support/TestStore.cs

@@ -0,0 +1,157 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Globalization;
+using System.Linq.Expressions;
+using Microsoft.Extensions.VectorData;
+
+namespace VectorData.ConformanceTests.Support;
+
+#pragma warning disable CA1001 // Type owns disposable fields but is not disposable
+
+public abstract class TestStore
+{
+    private readonly SemaphoreSlim _lock = new(1, 1);
+    private int _referenceCount;
+    private VectorStore? _defaultVectorStore;
+
+    /// <summary>
+    /// Some databases modify vectors on upsert, e.g. normalizing them, so vectors
+    /// returned cannot be compared with the original ones.
+    /// </summary>
+    public virtual bool VectorsComparable => true;
+
+    /// <summary>
+    /// Whether the database supports filtering by score threshold in vector search.
+    /// </summary>
+    public virtual bool SupportsScoreThreshold => true;
+
+    public virtual string DefaultDistanceFunction => DistanceFunction.CosineSimilarity;
+    public virtual string DefaultIndexKind => IndexKind.Flat;
+
+    protected abstract Task StartAsync();
+
+    protected virtual Task StopAsync()
+        => Task.CompletedTask;
+
+    public VectorStore DefaultVectorStore
+    {
+        get => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized");
+        set => this._defaultVectorStore = value;
+    }
+
+    public virtual async Task ReferenceCountingStartAsync()
+    {
+        await this._lock.WaitAsync();
+        try
+        {
+            if (this._referenceCount++ == 0)
+            {
+                await this.StartAsync();
+            }
+        }
+        finally
+        {
+            this._lock.Release();
+        }
+    }
+
+    public virtual async Task ReferenceCountingStopAsync()
+    {
+        await this._lock.WaitAsync();
+        try
+        {
+            if (--this._referenceCount == 0)
+            {
+                await this.StopAsync();
+                this._defaultVectorStore?.Dispose();
+            }
+        }
+        finally
+        {
+            this._lock.Release();
+        }
+    }
+
+    public virtual TKey GenerateKey<TKey>(int value)
+        => typeof(TKey) switch
+        {
+            _ when typeof(TKey) == typeof(int) => (TKey)(object)value,
+            _ when typeof(TKey) == typeof(long) => (TKey)(object)(long)value,
+            _ when typeof(TKey) == typeof(ulong) => (TKey)(object)(ulong)value,
+            _ when typeof(TKey) == typeof(string) => (TKey)(object)value.ToString(CultureInfo.InvariantCulture),
+            _ when typeof(TKey) == typeof(Guid) => (TKey)(object)new Guid($"00000000-0000-0000-0000-00{value:0000000000}"),
+
+            _ => throw new NotSupportedException($"Unsupported key of type '{typeof(TKey).Name}', override {nameof(TestStore)}.{nameof(this.GenerateKey)}")
+        };
+
+    /// <summary>
+    /// Applies any provider-specific rules to collection names (e.g. all-lowercase).
+    /// </summary>
+    /// <param name="baseName"></param>
+    /// <returns></returns>
+    public virtual string AdjustCollectionName(string baseName)
+        => baseName;
+
+    /// <summary>
+    /// Creates a collection for the given name and definition.
+    /// Override this to provide provider-specific collection options (e.g., partition key configuration).
+    /// </summary>
+    public virtual VectorStoreCollection<TKey, TRecord> CreateCollection<TKey, TRecord>(
+        string name,
+        VectorStoreCollectionDefinition definition)
+        where TKey : notnull
+        where TRecord : class
+        => this.DefaultVectorStore.GetCollection<TKey, TRecord>(name, definition);
+
+    /// <summary>
+    /// Creates a dynamic collection for the given name and definition.
+    /// Override this to provide provider-specific collection options (e.g., partition key configuration).
+    /// </summary>
+    public virtual VectorStoreCollection<object, Dictionary<string, object?>> CreateDynamicCollection(
+        string name,
+        VectorStoreCollectionDefinition definition)
+        => this.DefaultVectorStore.GetDynamicCollection(name, definition);
+
+    /// <summary>Loops until the expected number of records is visible in the given collection.</summary>
+    /// <remarks>Some databases upsert asynchronously, meaning that our seed data may not be visible immediately to tests.</remarks>
+    public virtual async Task WaitForDataAsync<TKey, TRecord>(
+        VectorStoreCollection<TKey, TRecord> collection,
+        int recordCount,
+        Expression<Func<TRecord, bool>>? filter = null,
+        Expression<Func<TRecord, object?>>? vectorProperty = null,
+        int? vectorSize = null,
+        object? dummyVector = null)
+        where TKey : notnull
+        where TRecord : class
+    {
+        if (vectorSize is not null && dummyVector is not null)
+        {
+            throw new ArgumentException("vectorSize or dummyVector can't both be set");
+        }
+
+        var vector = dummyVector ?? new ReadOnlyMemory<float>(Enumerable.Range(0, vectorSize ?? 3).Select(i => (float)i).ToArray());
+
+        for (var i = 0; i < 200; i++)
+        {
+            // Note that we very intentionally use SearchAsync and not filtering GetAsync, as we want to wait until the data is visible
+            // specifically via vector search (some databases may show data via filtering before they are indexed for vector search).
+            var results = collection.SearchAsync(
+                vector,
+                top: recordCount is 0 ? 1 : recordCount,
+                new()
+                {
+                    Filter = filter,
+                    VectorProperty = vectorProperty
+                });
+            var count = await results.CountAsync();
+            if (count == recordCount)
+            {
+                return;
+            }
+
+            await Task.Delay(TimeSpan.FromMilliseconds(100));
+        }
+
+        throw new InvalidOperationException("Data did not appear in the collection within the expected time.");
+    }
+}

dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixture.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace VectorData.ConformanceTests.Support;
+
+public abstract class VectorStoreCollectionFixture<TKey, TRecord> : VectorStoreCollectionFixtureBase<TKey, TRecord>
+    where TKey : notnull
+    where TRecord : TestRecord<TKey>
+{
+    public virtual async Task ReseedAsync()
+    {
+        // TODO: Use filtering delete, https://github.com/microsoft/semantic-kernel/issues/11830
+
+        const int BatchSize = 100;
+
+        TKey[] keys;
+        do
+        {
+            keys = await this.Collection.GetAsync(r => true, top: BatchSize).Select(r => r.Key).ToArrayAsync();
+            await this.Collection.DeleteAsync(keys);
+        } while (keys.Length == BatchSize);
+
+        await this.SeedAsync();
+    }
+}

dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreCollectionFixtureBase.cs

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+
+namespace VectorData.ConformanceTests.Support;
+
+#pragma warning disable CA1721 // Property names should not match get methods
+
+/// <summary>
+/// A test fixture that sets up a single collection in the test vector store, with a specific record definition
+/// and test data.
+/// </summary>
+public abstract class VectorStoreCollectionFixtureBase<TKey, TRecord> : VectorStoreFixture
+    where TKey : notnull
+    where TRecord : class
+{
+    private List<TRecord>? _testData;
+
+    public abstract VectorStoreCollectionDefinition CreateRecordDefinition();
+    protected virtual List<TRecord> BuildTestData() => [];
+
+    /// <summary>
+    /// The base name for the test collection used in tests, before any provider-specific collection naming rules have been applied.
+    /// </summary>
+    // TODO: Make protected
+    protected abstract string CollectionNameBase { get; }
+
+    /// <summary>
+    /// The actual name of the test collection, after any provider-specific collection naming rules have been applied.
+    /// </summary>
+    public virtual string CollectionName => this.TestStore.AdjustCollectionName(this.CollectionNameBase);
+
+    protected virtual string DistanceFunction => this.TestStore.DefaultDistanceFunction;
+    protected virtual string IndexKind => this.TestStore.DefaultIndexKind;
+
+    protected virtual VectorStoreCollection<TKey, TRecord> GetCollection()
+        => this.TestStore.CreateCollection<TKey, TRecord>(this.CollectionName, this.CreateRecordDefinition());
+
+    public override async Task InitializeAsync()
+    {
+        await base.InitializeAsync();
+
+        this.Collection = this.GetCollection();
+
+        if (await this.Collection.CollectionExistsAsync())
+        {
+            await this.Collection.EnsureCollectionDeletedAsync();
+        }
+
+        await this.Collection.EnsureCollectionExistsAsync();
+        await this.SeedAsync();
+    }
+
+    public virtual VectorStoreCollection<TKey, TRecord> Collection { get; private set; } = null!;
+
+    public List<TRecord> TestData => this._testData ??= this.BuildTestData();
+
+    protected virtual async Task SeedAsync()
+    {
+        if (this.TestData.Count > 0)
+        {
+            await this.Collection.UpsertAsync(this.TestData);
+            await this.WaitForDataAsync();
+        }
+    }
+
+    protected virtual Task WaitForDataAsync()
+        => this.TestStore.WaitForDataAsync(this.Collection, recordCount: this.TestData.Count);
+}

dotnet/test/VectorData/VectorData.ConformanceTests/Support/VectorStoreFixture.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using Xunit;
+
+namespace VectorData.ConformanceTests.Support;
+
+public abstract class VectorStoreFixture : IAsyncLifetime
+{
+    private int _nextKeyValue = 1;
+
+    public abstract TestStore TestStore { get; }
+    public virtual VectorStore VectorStore => this.TestStore.DefaultVectorStore;
+
+    public virtual string DefaultDistanceFunction => this.TestStore.DefaultDistanceFunction;
+    public virtual string DefaultIndexKind => this.TestStore.DefaultIndexKind;
+
+    public virtual Task InitializeAsync()
+        => this.TestStore.ReferenceCountingStartAsync();
+
+    public virtual Task DisposeAsync()
+        => this.TestStore.ReferenceCountingStopAsync();
+
+    public virtual TKey GenerateNextKey<TKey>()
+        => this.TestStore.GenerateKey<TKey>(Interlocked.Increment(ref this._nextKeyValue));
+
+    /// <summary>
+    /// Creates a collection for the given name and definition.
+    /// Delegates to <see cref="TestStore.CreateCollection{TKey, TRecord}"/> which can be overridden for provider-specific options.
+    /// </summary>
+    public virtual VectorStoreCollection<TKey, TRecord> CreateCollection<TKey, TRecord>(string name, VectorStoreCollectionDefinition definition)
+        where TKey : notnull
+        where TRecord : class
+        => this.TestStore.CreateCollection<TKey, TRecord>(name, definition);
+
+    /// <summary>
+    /// Creates a dynamic collection for the given name and definition.
+    /// Delegates to <see cref="TestStore.CreateDynamicCollection"/> which can be overridden for provider-specific options.
+    /// </summary>
+    public virtual VectorStoreCollection<object, Dictionary<string, object?>> CreateDynamicCollection(string name, VectorStoreCollectionDefinition definition)
+        => this.TestStore.CreateDynamicCollection(name, definition);
+}