diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index a1f32f6286a9..a3bc4f9a380d 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -320,7 +320,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Qdrant.UnitTests EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StepwisePlannerMigration", "samples\Demos\StepwisePlannerMigration\StepwisePlannerMigration.csproj", "{38374C62-0263-4FE8-A18C-70FC8132912B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBMongoDB.UnitTests", "src\Connectors\Connectors.AzureCosmosDBMongoDB.UnitTests\Connectors.AzureCosmosDBMongoDB.UnitTests.csproj", "{2918478E-BC86-4D53-9D01-9C318F80C14F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AIModelRouter", "samples\Demos\AIModelRouter\AIModelRouter.csproj", "{E06818E3-00A5-41AC-97ED-9491070CDEA1}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.AzureCosmosDBNoSQL.UnitTests", "src\Connectors\Connectors.AzureCosmosDBNoSQL.UnitTests\Connectors.AzureCosmosDBNoSQL.UnitTests.csproj", "{385A8FE5-87E2-4458-AE09-35E10BD2E67F}" EndProject @@ -799,6 +801,12 @@ Global {38374C62-0263-4FE8-A18C-70FC8132912B}.Publish|Any CPU.Build.0 = Debug|Any CPU {38374C62-0263-4FE8-A18C-70FC8132912B}.Release|Any CPU.ActiveCfg = Release|Any CPU {38374C62-0263-4FE8-A18C-70FC8132912B}.Release|Any CPU.Build.0 = Release|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Publish|Any CPU.Build.0 = Debug|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2918478E-BC86-4D53-9D01-9C318F80C14F}.Release|Any CPU.Build.0 = Release|Any CPU {E06818E3-00A5-41AC-97ED-9491070CDEA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E06818E3-00A5-41AC-97ED-9491070CDEA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {E06818E3-00A5-41AC-97ED-9491070CDEA1}.Publish|Any CPU.ActiveCfg = Debug|Any CPU @@ -920,6 +928,7 @@ Global {1D4667B9-9381-4E32-895F-123B94253EE8} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {E92AE954-8F3A-4A6F-A4F9-DC12017E5AAF} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {38374C62-0263-4FE8-A18C-70FC8132912B} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263} + {2918478E-BC86-4D53-9D01-9C318F80C14F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} {E06818E3-00A5-41AC-97ED-9491070CDEA1} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263} {385A8FE5-87E2-4458-AE09-35E10BD2E67F} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} EndGlobalSection diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/.editorconfig b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/.editorconfig new file mode 100644 index 000000000000..394eef685f21 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs new file mode 100644 index 000000000000..6d7db223d41b --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.SemanticKernel.Data; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +public class AzureCosmosDBMongoDBHotelModel(string hotelId) +{ + /// The key of the record. + [VectorStoreRecordKey] + public string HotelId { get; init; } = hotelId; + + /// A string metadata field. + [VectorStoreRecordData] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData] + public string? Description { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBKernelBuilderExtensionsTests.cs new file mode 100644 index 000000000000..86b64257988f --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBKernelBuilderExtensionsTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class AzureCosmosDBMongoDBKernelBuilderExtensionsTests +{ + private readonly IKernelBuilder _kernelBuilder = Kernel.CreateBuilder(); + + [Fact] + public void AddVectorStoreRegistersClass() + { + // Arrange + this._kernelBuilder.Services.AddSingleton(Mock.Of()); + + // Act + this._kernelBuilder.AddAzureCosmosDBMongoDBVectorStore(); + + var kernel = this._kernelBuilder.Build(); + var vectorStore = kernel.Services.GetRequiredService(); + + // Assert + Assert.NotNull(vectorStore); + Assert.IsType(vectorStore); + } +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBServiceCollectionExtensionsTests.cs new file mode 100644 index 000000000000..4e07f71b1d3a --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBServiceCollectionExtensionsTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class AzureCosmosDBMongoDBServiceCollectionExtensionsTests +{ + private readonly IServiceCollection _serviceCollection = new ServiceCollection(); + + [Fact] + public void AddVectorStoreRegistersClass() + { + // Arrange + this._serviceCollection.AddSingleton(Mock.Of()); + + // Act + this._serviceCollection.AddAzureCosmosDBMongoDBVectorStore(); + + var serviceProvider = this._serviceCollection.BuildServiceProvider(); + var vectorStore = serviceProvider.GetRequiredService(); + + // Assert + Assert.NotNull(vectorStore); + Assert.IsType(vectorStore); + } +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..ee5f74d79ddd --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs @@ -0,0 +1,651 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class AzureCosmosDBMongoDBVectorStoreRecordCollectionTests +{ + private readonly Mock _mockMongoDatabase = new(); + private readonly Mock> _mockMongoCollection = new(); + + public AzureCosmosDBMongoDBVectorStoreRecordCollectionTests() + { + this._mockMongoDatabase + .Setup(l => l.GetCollection(It.IsAny(), It.IsAny())) + .Returns(this._mockMongoCollection.Object); + } + + [Fact] + public void ConstructorForModelWithoutKeyThrowsException() + { + // Act & Assert + var exception = Assert.Throws(() => new AzureCosmosDBMongoDBVectorStoreRecordCollection(this._mockMongoDatabase.Object, "collection")); + Assert.Contains("No key property found", exception.Message); + } + + [Fact] + public void ConstructorWithDeclarativeModelInitializesCollection() + { + // Act & Assert + var collection = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + Assert.NotNull(collection); + } + + [Fact] + public void ConstructorWithImperativeModelInitializesCollection() + { + // Arrange + var definition = new VectorStoreRecordDefinition + { + Properties = [new VectorStoreRecordKeyProperty("Id", typeof(string))] + }; + + // Act + var collection = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + new() { VectorStoreRecordDefinition = definition }); + + // Assert + Assert.NotNull(collection); + } + + [Theory] + [MemberData(nameof(CollectionExistsData))] + public async Task CollectionExistsReturnsValidResultAsync(List collections, string collectionName, bool expectedResult) + { + // Arrange + var mockCursor = new Mock>(); + + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns(collections); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + collectionName); + + // Act + var actualResult = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(expectedResult, actualResult); + } + + [Theory] + [InlineData(true, 0)] + [InlineData(false, 1)] + public async Task CreateCollectionInvokesValidMethodsAsync(bool indexExists, int actualIndexCreations) + { + // Arrange + const string CollectionName = "collection"; + + List indexes = indexExists ? [new BsonDocument { ["name"] = "DescriptionEmbedding_" }] : []; + + var mockIndexCursor = new Mock>(); + mockIndexCursor + .SetupSequence(l => l.MoveNext(It.IsAny())) + .Returns(true) + .Returns(false); + + mockIndexCursor + .Setup(l => l.Current) + .Returns(indexes); + + var mockMongoIndexManager = new Mock>(); + + mockMongoIndexManager + .Setup(l => l.ListAsync(It.IsAny())) + .ReturnsAsync(mockIndexCursor.Object); + + this._mockMongoCollection + .Setup(l => l.Indexes) + .Returns(mockMongoIndexManager.Object); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(this._mockMongoDatabase.Object, CollectionName); + + // Act + await sut.CreateCollectionAsync(); + + // Assert + this._mockMongoDatabase.Verify(l => l.CreateCollectionAsync( + CollectionName, + It.IsAny(), + It.IsAny()), Times.Once()); + + this._mockMongoDatabase.Verify(l => l.RunCommandAsync( + It.Is>(command => + command.Document["createIndexes"] == CollectionName && + command.Document["indexes"].GetType() == typeof(BsonArray) && + ((BsonArray)command.Document["indexes"]).Count == 1), + It.IsAny(), + It.IsAny()), Times.Exactly(actualIndexCreations)); + } + + [Theory] + [MemberData(nameof(CreateCollectionIfNotExistsData))] + public async Task CreateCollectionIfNotExistsInvokesValidMethodsAsync(List collections, int actualCollectionCreations) + { + // Arrange + const string CollectionName = "collection"; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns(collections); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var mockIndexCursor = new Mock>(); + mockIndexCursor + .SetupSequence(l => l.MoveNext(It.IsAny())) + .Returns(true) + .Returns(false); + + mockIndexCursor + .Setup(l => l.Current) + .Returns([]); + + var mockMongoIndexManager = new Mock>(); + + mockMongoIndexManager + .Setup(l => l.ListAsync(It.IsAny())) + .ReturnsAsync(mockIndexCursor.Object); + + this._mockMongoCollection + .Setup(l => l.Indexes) + .Returns(mockMongoIndexManager.Object); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + CollectionName); + + // Act + await sut.CreateCollectionIfNotExistsAsync(); + + // Assert + this._mockMongoDatabase.Verify(l => l.CreateCollectionAsync( + CollectionName, + It.IsAny(), + It.IsAny()), Times.Exactly(actualCollectionCreations)); + } + + [Fact] + public async Task DeleteInvokesValidMethodsAsync() + { + // Arrange + const string RecordKey = "key"; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], RecordKey); + + // Act + await sut.DeleteAsync(RecordKey); + + // Assert + this._mockMongoCollection.Verify(l => l.DeleteOneAsync( + It.Is>(definition => + definition.Render(documentSerializer, serializerRegistry) == + expectedDefinition.Render(documentSerializer, serializerRegistry)), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task DeleteBatchInvokesValidMethodsAsync() + { + // Arrange + List recordKeys = ["key1", "key2"]; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.In(document => document["_id"].AsString, recordKeys); + + // Act + await sut.DeleteBatchAsync(recordKeys); + + // Assert + this._mockMongoCollection.Verify(l => l.DeleteManyAsync( + It.Is>(definition => + definition.Render(documentSerializer, serializerRegistry) == + expectedDefinition.Render(documentSerializer, serializerRegistry)), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task DeleteCollectionInvokesValidMethodsAsync() + { + // Arrange + const string CollectionName = "collection"; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + CollectionName); + + // Act + await sut.DeleteCollectionAsync(); + + // Assert + this._mockMongoDatabase.Verify(l => l.DropCollectionAsync( + It.Is(name => name == CollectionName), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task GetReturnsValidRecordAsync() + { + // Arrange + const string RecordKey = "key"; + + var document = new BsonDocument { ["_id"] = RecordKey, ["HotelName"] = "Test Name" }; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns([document]); + + this._mockMongoCollection + .Setup(l => l.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var result = await sut.GetAsync(RecordKey); + + // Assert + Assert.NotNull(result); + Assert.Equal(RecordKey, result.HotelId); + Assert.Equal("Test Name", result.HotelName); + } + + [Fact] + public async Task GetBatchReturnsValidRecordAsync() + { + // Arrange + var document1 = new BsonDocument { ["_id"] = "key1", ["HotelName"] = "Test Name 1" }; + var document2 = new BsonDocument { ["_id"] = "key2", ["HotelName"] = "Test Name 2" }; + var document3 = new BsonDocument { ["_id"] = "key3", ["HotelName"] = "Test Name 3" }; + + var mockCursor = new Mock>(); + mockCursor + .SetupSequence(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true) + .ReturnsAsync(false); + + mockCursor + .Setup(l => l.Current) + .Returns([document1, document2, document3]); + + this._mockMongoCollection + .Setup(l => l.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var results = await sut.GetBatchAsync(["key1", "key2", "key3"]).ToListAsync(); + + // Assert + Assert.NotNull(results[0]); + Assert.Equal("key1", results[0].HotelId); + Assert.Equal("Test Name 1", results[0].HotelName); + + Assert.NotNull(results[1]); + Assert.Equal("key2", results[1].HotelId); + Assert.Equal("Test Name 2", results[1].HotelName); + + Assert.NotNull(results[2]); + Assert.Equal("key3", results[2].HotelId); + Assert.Equal("Test Name 3", results[2].HotelName); + } + + [Fact] + public async Task UpsertReturnsRecordKeyAsync() + { + // Arrange + var hotel = new AzureCosmosDBMongoDBHotelModel("key") { HotelName = "Test Name" }; + + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], "key"); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var result = await sut.UpsertAsync(hotel); + + // Assert + Assert.Equal("key", result); + + this._mockMongoCollection.Verify(l => l.ReplaceOneAsync( + It.Is>(definition => + definition.Render(documentSerializer, serializerRegistry) == + expectedDefinition.Render(documentSerializer, serializerRegistry)), + It.Is(document => + document["_id"] == "key" && + document["HotelName"] == "Test Name"), + It.IsAny(), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task UpsertBatchReturnsRecordKeysAsync() + { + // Arrange + var hotel1 = new AzureCosmosDBMongoDBHotelModel("key1") { HotelName = "Test Name 1" }; + var hotel2 = new AzureCosmosDBMongoDBHotelModel("key2") { HotelName = "Test Name 2" }; + var hotel3 = new AzureCosmosDBMongoDBHotelModel("key3") { HotelName = "Test Name 3" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection"); + + // Act + var results = await sut.UpsertBatchAsync([hotel1, hotel2, hotel3]).ToListAsync(); + + // Assert + Assert.NotNull(results); + Assert.Equal(3, results.Count); + + Assert.Equal("key1", results[0]); + Assert.Equal("key2", results[1]); + Assert.Equal("key3", results[2]); + } + + [Fact] + public async Task UpsertWithModelWorksCorrectlyAsync() + { + var definition = new VectorStoreRecordDefinition + { + Properties = new List + { + new VectorStoreRecordKeyProperty("Id", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)) + } + }; + + await this.TestUpsertWithModeAsync( + dataModel: new TestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "HotelName", + definition: definition); + } + + [Fact] + public async Task UpsertWithVectorStoreModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModeAsync( + dataModel: new VectorStoreTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "hotel_name"); + } + + [Fact] + public async Task UpsertWithBsonModelWorksCorrectlyAsync() + { + var definition = new VectorStoreRecordDefinition + { + Properties = new List + { + new VectorStoreRecordKeyProperty("Id", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)) + } + }; + + await this.TestUpsertWithModeAsync( + dataModel: new BsonTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "hotel_name", + definition: definition); + } + + [Fact] + public async Task UpsertWithBsonVectorStoreModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModeAsync( + dataModel: new BsonVectorStoreTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "hotel_name"); + } + + [Fact] + public async Task UpsertWithBsonVectorStoreWithNameModelWorksCorrectlyAsync() + { + await this.TestUpsertWithModeAsync( + dataModel: new BsonVectorStoreWithNameTestModel { Id = "key", HotelName = "Test Name" }, + expectedPropertyName: "bson_hotel_name"); + } + + [Fact] + public async Task UpsertWithCustomMapperWorksCorrectlyAsync() + { + // Arrange + var hotel = new AzureCosmosDBMongoDBHotelModel("key") { HotelName = "Test Name" }; + + var mockMapper = new Mock>(); + + mockMapper + .Setup(l => l.MapFromDataToStorageModel(It.IsAny())) + .Returns(new BsonDocument { ["_id"] = "key", ["my_name"] = "Test Name" }); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + new() { BsonDocumentCustomMapper = mockMapper.Object }); + + // Act + var result = await sut.UpsertAsync(hotel); + + // Assert + Assert.Equal("key", result); + + this._mockMongoCollection.Verify(l => l.ReplaceOneAsync( + It.IsAny>(), + It.Is(document => + document["_id"] == "key" && + document["my_name"] == "Test Name"), + It.IsAny(), + It.IsAny()), Times.Once()); + } + + [Fact] + public async Task GetWithCustomMapperWorksCorrectlyAsync() + { + // Arrange + const string RecordKey = "key"; + + var document = new BsonDocument { ["_id"] = RecordKey, ["my_name"] = "Test Name" }; + + var mockCursor = new Mock>(); + mockCursor + .Setup(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true); + + mockCursor + .Setup(l => l.Current) + .Returns([document]); + + this._mockMongoCollection + .Setup(l => l.FindAsync( + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var mockMapper = new Mock>(); + + mockMapper + .Setup(l => l.MapFromStorageToDataModel(It.IsAny(), It.IsAny())) + .Returns(new AzureCosmosDBMongoDBHotelModel(RecordKey) { HotelName = "Name from mapper" }); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + new() { BsonDocumentCustomMapper = mockMapper.Object }); + + // Act + var result = await sut.GetAsync(RecordKey); + + // Assert + Assert.NotNull(result); + Assert.Equal(RecordKey, result.HotelId); + Assert.Equal("Name from mapper", result.HotelName); + } + + public static TheoryData, string, bool> CollectionExistsData => new() + { + { ["collection-2"], "collection-2", true }, + { [], "non-existent-collection", false } + }; + + public static TheoryData, int> CreateCollectionIfNotExistsData => new() + { + { ["collection"], 0 }, + { [], 1 } + }; + + #region private + + private async Task TestUpsertWithModeAsync( + TDataModel dataModel, + string expectedPropertyName, + VectorStoreRecordDefinition? definition = null) + where TDataModel : class + { + // Arrange + var serializerRegistry = BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer(); + var expectedDefinition = Builders.Filter.Eq(document => document["_id"], "key"); + + AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions? options = definition != null ? + new() { VectorStoreRecordDefinition = definition } : + null; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + options); + + // Act + var result = await sut.UpsertAsync(dataModel); + + // Assert + Assert.Equal("key", result); + + this._mockMongoCollection.Verify(l => l.ReplaceOneAsync( + It.Is>(definition => + definition.Render(documentSerializer, serializerRegistry) == + expectedDefinition.Render(documentSerializer, serializerRegistry)), + It.Is(document => + document["_id"] == "key" && + document.Contains(expectedPropertyName) && + document[expectedPropertyName] == "Test Name"), + It.IsAny(), + It.IsAny()), Times.Once()); + } + +#pragma warning disable CA1812 + private sealed class TestModel + { + public string? Id { get; set; } + + public string? HotelName { get; set; } + } + + private sealed class VectorStoreTestModel + { + [VectorStoreRecordKey] + public string? Id { get; set; } + + [VectorStoreRecordData(StoragePropertyName = "hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonTestModel + { + [BsonId] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreTestModel + { + [BsonId] + [VectorStoreRecordKey] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + [VectorStoreRecordData] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreWithNameTestModel + { + [BsonId] + [VectorStoreRecordKey] + public string? Id { get; set; } + + [BsonElement("bson_hotel_name")] + [VectorStoreRecordData(StoragePropertyName = "storage_hotel_name")] + public string? HotelName { get; set; } + } +#pragma warning restore CA1812 + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordMapperTests.cs new file mode 100644 index 000000000000..e561b6f32d4e --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordMapperTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; +using Xunit; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class AzureCosmosDBMongoDBVectorStoreRecordMapperTests +{ + private readonly AzureCosmosDBMongoDBVectorStoreRecordMapper _sut; + + public AzureCosmosDBMongoDBVectorStoreRecordMapperTests() + { + var definition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("HotelId", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)), + new VectorStoreRecordDataProperty("Tags", typeof(List)), + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory?)) { StoragePropertyName = "description_embedding " } + ] + }; + + var storagePropertyNames = new Dictionary + { + ["HotelId"] = "HotelId", + ["HotelName"] = "HotelName", + ["Tags"] = "Tags", + ["DescriptionEmbedding"] = "description_embedding", + }; + + this._sut = new(definition, storagePropertyNames); + } + + [Fact] + public void MapFromDataToStorageModelReturnsValidObject() + { + // Arrange + var hotel = new AzureCosmosDBMongoDBHotelModel("key") + { + HotelName = "Test Name", + Tags = ["tag1", "tag2"], + DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) + }; + + // Act + var document = this._sut.MapFromDataToStorageModel(hotel); + + // Assert + Assert.NotNull(document); + + Assert.Equal("key", document["_id"]); + Assert.Equal("Test Name", document["HotelName"]); + Assert.Equal(["tag1", "tag2"], document["Tags"].AsBsonArray); + Assert.Equal([1f, 2f, 3f], document["description_embedding"].AsBsonArray); + } + + [Fact] + public void MapFromStorageToDataModelReturnsValidObject() + { + // Arrange + var document = new BsonDocument + { + ["_id"] = "key", + ["HotelName"] = "Test Name", + ["Tags"] = BsonArray.Create(new List { "tag1", "tag2" }), + ["description_embedding"] = BsonArray.Create(new List { 1f, 2f, 3f }) + }; + + // Act + var hotel = this._sut.MapFromStorageToDataModel(document, new()); + + // Assert + Assert.NotNull(hotel); + + Assert.Equal("key", hotel.HotelId); + Assert.Equal("Test Name", hotel.HotelName); + Assert.Equal(["tag1", "tag2"], hotel.Tags); + Assert.True(new ReadOnlyMemory([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span)); + } +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreTests.cs new file mode 100644 index 000000000000..3bc2049bf2c9 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreTests.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; +using Moq; +using Xunit; + +namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests; + +/// +/// Unit tests for class. +/// +public sealed class AzureCosmosDBMongoDBVectorStoreTests +{ + private readonly Mock _mockMongoDatabase = new(); + + [Fact] + public void GetCollectionWithNotSupportedKeyThrowsException() + { + // Arrange + var sut = new AzureCosmosDBMongoDBVectorStore(this._mockMongoDatabase.Object); + + // Act & Assert + Assert.Throws(() => sut.GetCollection("collection")); + } + + [Fact] + public void GetCollectionWithFactoryReturnsCustomCollection() + { + // Arrange + var mockFactory = new Mock(); + var mockRecordCollection = new Mock>(); + + mockFactory + .Setup(l => l.CreateVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + It.IsAny())) + .Returns(mockRecordCollection.Object); + + var sut = new AzureCosmosDBMongoDBVectorStore( + this._mockMongoDatabase.Object, + new AzureCosmosDBMongoDBVectorStoreOptions { VectorStoreCollectionFactory = mockFactory.Object }); + + // Act + var collection = sut.GetCollection("collection"); + + // Assert + Assert.Same(mockRecordCollection.Object, collection); + mockFactory.Verify(l => l.CreateVectorStoreRecordCollection( + this._mockMongoDatabase.Object, + "collection", + It.IsAny()), Times.Once()); + } + + [Fact] + public void GetCollectionWithoutFactoryReturnsDefaultCollection() + { + // Arrange + var sut = new AzureCosmosDBMongoDBVectorStore(this._mockMongoDatabase.Object); + + // Act + var collection = sut.GetCollection("collection"); + + // Assert + Assert.NotNull(collection); + } + + [Fact] + public async Task ListCollectionNamesReturnsCollectionNamesAsync() + { + // Arrange + var expectedCollectionNames = new List { "collection-1", "collection-2", "collection-3" }; + + var mockCursor = new Mock>(); + mockCursor + .SetupSequence(l => l.MoveNextAsync(It.IsAny())) + .ReturnsAsync(true) + .ReturnsAsync(false); + + mockCursor + .Setup(l => l.Current) + .Returns(expectedCollectionNames); + + this._mockMongoDatabase + .Setup(l => l.ListCollectionNamesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(mockCursor.Object); + + var sut = new AzureCosmosDBMongoDBVectorStore(this._mockMongoDatabase.Object); + + // Act + var actualCollectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(expectedCollectionNames, actualCollectionNames); + } +} diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/Connectors.AzureCosmosDBMongoDB.UnitTests.csproj b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/Connectors.AzureCosmosDBMongoDB.UnitTests.csproj new file mode 100644 index 000000000000..a31e4b802b52 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/Connectors.AzureCosmosDBMongoDB.UnitTests.csproj @@ -0,0 +1,32 @@ + + + + SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests + SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests + net8.0 + true + enable + disable + false + $(NoWarn);SKEXP0001,SKEXP0020 + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBConstants.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBConstants.cs new file mode 100644 index 000000000000..a5bf87d1a960 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBConstants.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Constants for Azure CosmosDB MongoDB vector store implementation. +/// +internal static class AzureCosmosDBMongoDBConstants +{ + /// Reserved key property name in Azure CosmosDB MongoDB. + internal const string MongoReservedKeyPropertyName = "_id"; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBKernelBuilderExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBKernelBuilderExtensions.cs new file mode 100644 index 000000000000..807bb030dcfc --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBKernelBuilderExtensions.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; + +namespace Microsoft.SemanticKernel; + +/// +/// Extension methods to register Azure CosmosDB MongoDB instances on the . +/// +public static class AzureCosmosDBMongoDBKernelBuilderExtensions +{ + /// + /// Register a Azure CosmosDB MongoDB with the specified service ID + /// and where the Azure CosmosDB MongoDB is retrieved from the dependency injection container. + /// + /// The builder to register the on. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// The kernel builder. + public static IKernelBuilder AddAzureCosmosDBMongoDBVectorStore( + this IKernelBuilder builder, + AzureCosmosDBMongoDBVectorStoreOptions? options = default, + string? serviceId = default) + { + builder.Services.AddAzureCosmosDBMongoDBVectorStore(options, serviceId); + return builder; + } + + /// + /// Register a Azure CosmosDB MongoDB with the specified service ID + /// and where the Azure CosmosDB MongoDB is constructed using the provided and . + /// + /// The builder to register the on. + /// Connection string required to connect to Azure CosmosDB MongoDB. + /// Database name for Azure CosmosDB MongoDB. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// The kernel builder. + public static IKernelBuilder AddAzureCosmosDBMongoDBVectorStore( + this IKernelBuilder builder, + string connectionString, + string databaseName, + AzureCosmosDBMongoDBVectorStoreOptions? options = default, + string? serviceId = default) + { + builder.Services.AddAzureCosmosDBMongoDBVectorStore(connectionString, databaseName, options, serviceId); + return builder; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBNamingConvention.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBNamingConvention.cs new file mode 100644 index 000000000000..bece10b432d6 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBNamingConvention.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Conventions; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Naming convention for storage properties based on provided name mapping. +/// +internal sealed class AzureCosmosDBMongoDBNamingConvention(IReadOnlyDictionary nameMapping) : IMemberMapConvention +{ + private readonly IReadOnlyDictionary _nameMapping = nameMapping; + + public string Name => nameof(AzureCosmosDBMongoDBNamingConvention); + + public void Apply(BsonMemberMap memberMap) + { + var memberName = memberMap.MemberName; + var name = this._nameMapping.TryGetValue(memberName, out var customName) ? customName : memberName; + + memberMap.SetElementName(name); + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBServiceCollectionExtensions.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBServiceCollectionExtensions.cs new file mode 100644 index 000000000000..02f26e85ee94 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBServiceCollectionExtensions.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; + +namespace Microsoft.SemanticKernel; + +/// +/// Extension methods to register Azure CosmosDB MongoDB instances on an . +/// +public static class AzureCosmosDBMongoDBServiceCollectionExtensions +{ + /// + /// Register a Azure CosmosDB MongoDB with the specified service ID + /// and where the Azure CosmosDB MongoDB is retrieved from the dependency injection container. + /// + /// The to register the on. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// Service collection. + public static IServiceCollection AddAzureCosmosDBMongoDBVectorStore( + this IServiceCollection services, + AzureCosmosDBMongoDBVectorStoreOptions? options = default, + string? serviceId = default) + { + // If we are not constructing MongoDatabase, add the IVectorStore as transient, since we + // cannot make assumptions about how MongoDatabase is being managed. + services.AddKeyedTransient( + serviceId, + (sp, obj) => + { + var database = sp.GetRequiredService(); + var selectedOptions = options ?? sp.GetService(); + + return new AzureCosmosDBMongoDBVectorStore(database, options); + }); + + return services; + } + + /// + /// Register a Azure CosmosDB MongoDB with the specified service ID + /// and where the Azure CosmosDB MongoDB is constructed using the provided and . + /// + /// The to register the on. + /// Connection string required to connect to Azure CosmosDB MongoDB. + /// Database name for Azure CosmosDB MongoDB. + /// Optional options to further configure the . + /// An optional service id to use as the service key. + /// Service collection. + public static IServiceCollection AddAzureCosmosDBMongoDBVectorStore( + this IServiceCollection services, + string connectionString, + string databaseName, + AzureCosmosDBMongoDBVectorStoreOptions? options = default, + string? serviceId = default) + { + // If we are constructing IMongoDatabase, add the IVectorStore as singleton, since we are managing the lifetime of it, + // and the recommendation from Mongo is to register it with a singleton lifetime. + services.AddKeyedSingleton( + serviceId, + (sp, obj) => + { + var settings = MongoClientSettings.FromConnectionString(connectionString); + var mongoClient = new MongoClient(settings); + var database = mongoClient.GetDatabase(databaseName); + + var selectedOptions = options ?? sp.GetService(); + + return new AzureCosmosDBMongoDBVectorStore(database, options); + }); + + return services; + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs new file mode 100644 index 000000000000..7f907d068983 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Class for accessing the list of collections in a Azure CosmosDB MongoDB vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class AzureCosmosDBMongoDBVectorStore : IVectorStore +{ + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + private readonly IMongoDatabase _mongoDatabase; + + /// Optional configuration options for this class. + private readonly AzureCosmosDBMongoDBVectorStoreOptions _options; + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + /// Optional configuration options for this class. + public AzureCosmosDBMongoDBVectorStore(IMongoDatabase mongoDatabase, AzureCosmosDBMongoDBVectorStoreOptions? options = default) + { + Verify.NotNull(mongoDatabase); + + this._mongoDatabase = mongoDatabase; + this._options = options ?? new(); + } + + /// + public IVectorStoreRecordCollection GetCollection(string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition = null) + where TKey : notnull + where TRecord : class + { + if (typeof(TKey) != typeof(string)) + { + throw new NotSupportedException("Only string keys are supported."); + } + + if (this._options.VectorStoreCollectionFactory is not null) + { + return this._options.VectorStoreCollectionFactory.CreateVectorStoreRecordCollection(this._mongoDatabase, name, vectorStoreRecordDefinition); + } + + var recordCollection = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + this._mongoDatabase, + name, + new() { VectorStoreRecordDefinition = vectorStoreRecordDefinition }) as IVectorStoreRecordCollection; + + return recordCollection!; + } + + /// + public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using var cursor = await this._mongoDatabase + .ListCollectionNamesAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var name in cursor.Current) + { + yield return name; + } + } + } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreOptions.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreOptions.cs new file mode 100644 index 000000000000..08df3aef81d8 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreOptions.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Options when creating a +/// +public sealed class AzureCosmosDBMongoDBVectorStoreOptions +{ + /// + /// An optional factory to use for constructing instances, if a custom record collection is required. + /// + public IAzureCosmosDBMongoDBVectorStoreRecordCollectionFactory? VectorStoreCollectionFactory { get; init; } +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs new file mode 100644 index 000000000000..2d409ef61c54 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs @@ -0,0 +1,431 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage. +/// +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class AzureCosmosDBMongoDBVectorStoreRecordCollection : IVectorStoreRecordCollection where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// The name of this database for telemetry purposes. + private const string DatabaseName = "AzureCosmosDBMongoDB"; + + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + private readonly IMongoDatabase _mongoDatabase; + + /// Azure CosmosDB MongoDB collection to perform record operations. + private readonly IMongoCollection _mongoCollection; + + /// Optional configuration options for this class. + private readonly AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions _options; + + /// A definition of the current storage model. + private readonly VectorStoreRecordDefinition _vectorStoreRecordDefinition; + + /// Interface for mapping between a storage model, and the consumer record data model. + private readonly IVectorStoreRecordMapper _mapper; + + /// A dictionary that maps from a property name to the storage name that should be used when serializing it for data and vector properties. + private readonly Dictionary _storagePropertyNames; + + /// Collection of vector storage property names. + private readonly List _vectorStoragePropertyNames; + + /// Collection of record vector properties. + private readonly List _vectorProperties; + + /// + public string CollectionName { get; } + + /// + /// Initializes a new instance of the class. + /// + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + public AzureCosmosDBMongoDBVectorStoreRecordCollection( + IMongoDatabase mongoDatabase, + string collectionName, + AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions? options = default) + { + // Verify. + Verify.NotNull(mongoDatabase); + Verify.NotNullOrWhiteSpace(collectionName); + + // Assign. + this._mongoDatabase = mongoDatabase; + this._mongoCollection = mongoDatabase.GetCollection(collectionName); + this.CollectionName = collectionName; + this._options = options ?? new AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions(); + this._vectorStoreRecordDefinition = this._options.VectorStoreRecordDefinition ?? VectorStoreRecordPropertyReader.CreateVectorStoreRecordDefinitionFromType(typeof(TRecord), true); + + var properties = VectorStoreRecordPropertyReader.SplitDefinitionAndVerify( + typeof(TRecord).Name, + this._vectorStoreRecordDefinition, + supportsMultipleVectors: true, + requiresAtLeastOneVector: false); + + this._storagePropertyNames = GetStoragePropertyNames(properties, typeof(TRecord)); + this._vectorProperties = properties.VectorProperties; + this._vectorStoragePropertyNames = this._vectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); + + this._mapper = this._options.BsonDocumentCustomMapper ?? + new AzureCosmosDBMongoDBVectorStoreRecordMapper(this._vectorStoreRecordDefinition, this._storagePropertyNames); + } + + /// + public Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => this.RunOperationAsync("ListCollectionNames", () => this.InternalCollectionExistsAsync(cancellationToken)); + + /// + public async Task CreateCollectionAsync(CancellationToken cancellationToken = default) + { + await this.RunOperationAsync("CreateCollection", + () => this._mongoDatabase.CreateCollectionAsync(this.CollectionName, cancellationToken: cancellationToken)).ConfigureAwait(false); + + await this.RunOperationAsync("CreateIndex", + () => this.CreateIndexAsync(this.CollectionName, cancellationToken: cancellationToken)).ConfigureAwait(false); + } + + /// + public async Task CreateCollectionIfNotExistsAsync(CancellationToken cancellationToken = default) + { + if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) + { + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + public async Task DeleteAsync(string key, DeleteRecordOptions? options = null, CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(key); + + await this.RunOperationAsync("DeleteOne", () => this._mongoCollection.DeleteOneAsync(this.GetFilterById(key), cancellationToken)) + .ConfigureAwait(false); + } + + /// + public async Task DeleteBatchAsync(IEnumerable keys, DeleteRecordOptions? options = null, CancellationToken cancellationToken = default) + { + Verify.NotNull(keys); + + await this.RunOperationAsync("DeleteMany", () => this._mongoCollection.DeleteManyAsync(this.GetFilterByIds(keys), cancellationToken)) + .ConfigureAwait(false); + } + + /// + public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + => this.RunOperationAsync("DropCollection", () => this._mongoDatabase.DropCollectionAsync(this.CollectionName, cancellationToken)); + + /// + public async Task GetAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(key); + + const string OperationName = "Find"; + + var record = await this.RunOperationAsync(OperationName, async () => + { + using var cursor = await this + .FindAsync(this.GetFilterById(key), options, cancellationToken) + .ConfigureAwait(false); + + return await cursor.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + + if (record is null) + { + return null; + } + + return VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + OperationName, + () => this._mapper.MapFromStorageToDataModel(record, new())); + } + + /// + public async IAsyncEnumerable GetBatchAsync( + IEnumerable keys, + GetRecordOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Verify.NotNull(keys); + + const string OperationName = "Find"; + + using var cursor = await this + .FindAsync(this.GetFilterByIds(keys), options, cancellationToken) + .ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var record in cursor.Current) + { + if (record is not null) + { + yield return VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + OperationName, + () => this._mapper.MapFromStorageToDataModel(record, new())); + } + } + } + } + + /// + public Task UpsertAsync(TRecord record, UpsertRecordOptions? options = null, CancellationToken cancellationToken = default) + { + Verify.NotNull(record); + + const string OperationName = "ReplaceOne"; + + var replaceOptions = new ReplaceOptions { IsUpsert = true }; + var storageModel = VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + OperationName, + () => this._mapper.MapFromDataToStorageModel(record)); + + var key = storageModel[AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName].AsString; + + return this.RunOperationAsync(OperationName, async () => + { + await this._mongoCollection + .ReplaceOneAsync(this.GetFilterById(key), storageModel, replaceOptions, cancellationToken) + .ConfigureAwait(false); + + return key; + }); + } + + /// + public async IAsyncEnumerable UpsertBatchAsync( + IEnumerable records, + UpsertRecordOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Verify.NotNull(records); + + var tasks = records.Select(record => this.UpsertAsync(record, options, cancellationToken)); + var results = await Task.WhenAll(tasks).ConfigureAwait(false); + + foreach (var result in results) + { + if (result is not null) + { + yield return result; + } + } + } + + #region private + + private async Task CreateIndexAsync(string collectionName, CancellationToken cancellationToken) + { + var indexCursor = await this._mongoCollection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = indexCursor.ToList(cancellationToken).Select(index => index["name"].ToString()) ?? []; + var uniqueIndexes = new HashSet(indexes); + + var indexArray = new BsonArray(); + + // Create separate index for each vector property + foreach (var property in this._vectorStoreRecordDefinition.Properties.OfType()) + { + // Use index name same as vector property name with underscore + var vectorPropertyName = this._storagePropertyNames[property.DataModelPropertyName]; + var indexName = $"{vectorPropertyName}_"; + + // If index already exists, proceed to the next vector property + if (uniqueIndexes.Contains(indexName)) + { + continue; + } + + // Otherwise, create a new index + var searchOptions = new BsonDocument + { + { "kind", GetIndexKind(property.IndexKind, vectorPropertyName) }, + { "numLists", this._options.NumLists }, + { "similarity", GetDistanceFunction(property.DistanceFunction, vectorPropertyName) }, + { "dimensions", property.Dimensions } + }; + + if (this._options.EfConstruction is not null) + { + searchOptions["efConstruction"] = this._options.EfConstruction; + } + + var indexDocument = new BsonDocument + { + ["name"] = indexName, + ["key"] = new BsonDocument { [vectorPropertyName] = "cosmosSearch" }, + ["cosmosSearchOptions"] = searchOptions + }; + + indexArray.Add(indexDocument); + } + + if (indexArray.Count > 0) + { + var createIndexCommand = new BsonDocument + { + { "createIndexes", collectionName }, + { "indexes", indexArray } + }; + + await this._mongoDatabase.RunCommandAsync(createIndexCommand, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + private async Task> FindAsync(FilterDefinition filter, GetRecordOptions? options, CancellationToken cancellationToken) + { + ProjectionDefinitionBuilder projectionBuilder = Builders.Projection; + ProjectionDefinition? projectionDefinition = null; + + var includeVectors = options?.IncludeVectors ?? false; + + if (!includeVectors && this._vectorStoragePropertyNames.Count > 0) + { + foreach (var vectorPropertyName in this._vectorStoragePropertyNames) + { + projectionDefinition = projectionDefinition is not null ? + projectionDefinition.Exclude(vectorPropertyName) : + projectionBuilder.Exclude(vectorPropertyName); + } + } + + var findOptions = projectionDefinition is not null ? + new FindOptions { Projection = projectionDefinition } : + null; + + return await this._mongoCollection.FindAsync(filter, findOptions, cancellationToken).ConfigureAwait(false); + } + + private FilterDefinition GetFilterById(string id) + => Builders.Filter.Eq(document => document[AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName], id); + + private FilterDefinition GetFilterByIds(IEnumerable ids) + => Builders.Filter.In(document => document[AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName].AsString, ids); + + private async Task InternalCollectionExistsAsync(CancellationToken cancellationToken) + { + var filter = new BsonDocument("name", this.CollectionName); + var options = new ListCollectionNamesOptions { Filter = filter }; + + using var cursor = await this._mongoDatabase.ListCollectionNamesAsync(options, cancellationToken: cancellationToken).ConfigureAwait(false); + + return await cursor.AnyAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task RunOperationAsync(string operationName, Func operation) + { + try + { + await operation.Invoke().ConfigureAwait(false); + } + catch (Exception ex) + { + throw new VectorStoreOperationException("Call to vector store failed.", ex) + { + VectorStoreType = DatabaseName, + CollectionName = this.CollectionName, + OperationName = operationName + }; + } + } + + private async Task RunOperationAsync(string operationName, Func> operation) + { + try + { + return await operation.Invoke().ConfigureAwait(false); + } + catch (Exception ex) + { + throw new VectorStoreOperationException("Call to vector store failed.", ex) + { + VectorStoreType = DatabaseName, + CollectionName = this.CollectionName, + OperationName = operationName + }; + } + } + + /// + /// More information about Azure CosmosDB for MongoDB index kinds here: . + /// + private static string GetIndexKind(string? indexKind, string vectorPropertyName) + { + return indexKind switch + { + IndexKind.Hnsw => "vector-hnsw", + IndexKind.IvfFlat => "vector-ivf", + _ => throw new InvalidOperationException($"Index kind '{indexKind}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.") + }; + } + + /// + /// More information about Azure CosmosDB for MongoDB distance functions here: . + /// + private static string GetDistanceFunction(string? distanceFunction, string vectorPropertyName) + { + return distanceFunction switch + { + DistanceFunction.CosineDistance => "COS", + DistanceFunction.DotProductSimilarity => "IP", + DistanceFunction.EuclideanDistance => "L2", + _ => throw new InvalidOperationException($"Distance function '{distanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.") + }; + } + + /// + /// Gets storage property names taking into account BSON serialization attributes. + /// + private static Dictionary GetStoragePropertyNames( + (VectorStoreRecordKeyProperty KeyProperty, List DataProperties, List VectorProperties) properties, + Type dataModel) + { + var storagePropertyNames = VectorStoreRecordPropertyReader.BuildPropertyNameToStorageNameMap(properties); + + var allProperties = new List([properties.KeyProperty]) + .Concat(properties.DataProperties) + .Concat(properties.VectorProperties); + + foreach (var property in allProperties) + { + var propertyInfo = dataModel.GetProperty(property.DataModelPropertyName); + + if (propertyInfo != null) + { + var bsonElementAttribute = propertyInfo.GetCustomAttribute(); + if (bsonElementAttribute is not null) + { + storagePropertyNames[property.DataModelPropertyName] = bsonElementAttribute.ElementName; + } + } + } + + return storagePropertyNames; + } + + #endregion +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions.cs new file mode 100644 index 000000000000..11b21a1e84e7 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Options when creating a . +/// +public sealed class AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions where TRecord : class +{ + /// + /// Gets or sets an optional custom mapper to use when converting between the data model and the Azure CosmosDB MongoDB BSON object. + /// + public IVectorStoreRecordMapper? BsonDocumentCustomMapper { get; init; } = null; + + /// + /// Gets or sets an optional record definition that defines the schema of the record type. + /// + /// + /// If not provided, the schema will be inferred from the record model class using reflection. + /// In this case, the record model properties must be annotated with the appropriate attributes to indicate their usage. + /// See , and . + /// + public VectorStoreRecordDefinition? VectorStoreRecordDefinition { get; init; } = null; + + /// + /// This integer is the number of clusters that the inverted file (IVF) index uses to group the vector data. Default is 1. + /// We recommend that numLists is set to documentCount/1000 for up to 1 million documents and to sqrt(documentCount) + /// for more than 1 million documents. Using a numLists value of 1 is akin to performing brute-force search, which has + /// limited performance. + /// + public int NumLists { get; set; } = 1; + + /// + /// The size of the dynamic candidate list for constructing the graph (64 by default, minimum value is 4, + /// maximum value is 1000). Higher ef_construction will result in better index quality and higher accuracy, but it will + /// also increase the time required to build the index. EfConstruction has to be at least 2 * m + /// + public int? EfConstruction { get; set; } = null; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordMapper.cs new file mode 100644 index 000000000000..5e23aa506956 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordMapper.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Conventions; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +internal sealed class AzureCosmosDBMongoDBVectorStoreRecordMapper : IVectorStoreRecordMapper + where TRecord : class +{ + /// A set of types that a key on the provided model may have. + private static readonly HashSet s_supportedKeyTypes = + [ + typeof(string) + ]; + + /// A set of types that data properties on the provided model may have. + private static readonly HashSet s_supportedDataTypes = + [ + typeof(bool), + typeof(bool?), + typeof(string), + typeof(int), + typeof(int?), + typeof(long), + typeof(long?), + typeof(float), + typeof(float?), + typeof(double), + typeof(double?), + typeof(decimal), + typeof(decimal?), + ]; + + /// A set of types that vectors on the provided model may have. + private static readonly HashSet s_supportedVectorTypes = + [ + typeof(ReadOnlyMemory), + typeof(ReadOnlyMemory?), + typeof(ReadOnlyMemory), + typeof(ReadOnlyMemory?) + ]; + + /// A dictionary that maps from a property name to the storage name. + private readonly Dictionary _storagePropertyNames; + + /// + /// Initializes a new instance of the class. + /// + /// The record definition that defines the schema of the record type. + /// A dictionary that maps from a property name to the configured name that should be used when storing it. + public AzureCosmosDBMongoDBVectorStoreRecordMapper(VectorStoreRecordDefinition vectorStoreRecordDefinition, Dictionary storagePropertyNames) + { + var (keyProperty, dataProperties, vectorProperties) = VectorStoreRecordPropertyReader.FindProperties(typeof(TRecord), vectorStoreRecordDefinition, supportsMultipleVectors: true); + + VectorStoreRecordPropertyReader.VerifyPropertyTypes([keyProperty], s_supportedKeyTypes, "Key"); + VectorStoreRecordPropertyReader.VerifyPropertyTypes(dataProperties, s_supportedDataTypes, "Data", supportEnumerable: true); + VectorStoreRecordPropertyReader.VerifyPropertyTypes(vectorProperties, s_supportedVectorTypes, "Vector"); + + this._storagePropertyNames = storagePropertyNames; + + // Use Mongo reserved key property name as storage key property name + this._storagePropertyNames[keyProperty.Name] = AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName; + + var conventionPack = new ConventionPack + { + new IgnoreExtraElementsConvention(ignoreExtraElements: true), + new AzureCosmosDBMongoDBNamingConvention(this._storagePropertyNames) + }; + + ConventionRegistry.Register( + nameof(AzureCosmosDBMongoDBVectorStoreRecordMapper), + conventionPack, + type => type == typeof(TRecord)); + } + + public BsonDocument MapFromDataToStorageModel(TRecord dataModel) + => dataModel.ToBsonDocument(); + + public TRecord MapFromStorageToDataModel(BsonDocument storageModel, StorageToDataModelMapperOptions options) + => BsonSerializer.Deserialize(storageModel); +} diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj index 747709f993cc..9ce9d24d1aed 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj @@ -23,6 +23,10 @@ + + + + diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/IAzureCosmosDBMongoDBVectorStoreRecordCollectionFactory.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/IAzureCosmosDBMongoDBVectorStoreRecordCollectionFactory.cs new file mode 100644 index 000000000000..39231a8bf7a8 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/IAzureCosmosDBMongoDBVectorStoreRecordCollectionFactory.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; + +namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; + +/// +/// Interface for constructing Azure CosmosDB MongoDB instances when using to retrieve these. +/// +public interface IAzureCosmosDBMongoDBVectorStoreRecordCollectionFactory +{ + /// + /// Constructs a new instance of the . + /// + /// The data type of the record key. + /// The data model to use for adding, updating and retrieving data from storage. + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + /// The name of the collection to connect to. + /// An optional record definition that defines the schema of the record type. If not present, attributes on will be used. + /// The new instance of . + IVectorStoreRecordCollection CreateVectorStoreRecordCollection(IMongoDatabase mongoDatabase, string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition) + where TKey : notnull + where TRecord : class; +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBMemoryStoreTestsFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBMemoryStoreTestsFixture.cs index 1b1255c46b68..6854e7e7fdf8 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBMemoryStoreTestsFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBMemoryStoreTestsFixture.cs @@ -54,7 +54,7 @@ public async Task DisposeAsync() private static string GetSetting(IConfigurationRoot configuration, string settingName) { - var settingValue = configuration[$"AzureCosmosDB:{settingName}"]; + var settingValue = configuration[$"AzureCosmosDBMongoDB:{settingName}"]; if (string.IsNullOrWhiteSpace(settingValue)) { throw new ArgumentNullException($"{settingValue} string is not configured"); diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreCollectionFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreCollectionFixture.cs new file mode 100644 index 000000000000..43f30eb5d520 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreCollectionFixture.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; + +[CollectionDefinition("AzureCosmosDBMongoDBVectorStoreCollection")] +public class AzureCosmosDBMongoDBVectorStoreCollectionFixture : ICollectionFixture +{ } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs new file mode 100644 index 000000000000..3af1a3c66b1a --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel.Data; +using MongoDB.Driver; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; + +public class AzureCosmosDBMongoDBVectorStoreFixture : IAsyncLifetime +{ + private readonly List _testCollections = ["sk-test-hotels", "sk-test-contacts", "sk-test-addresses"]; + + /// Main test collection for tests. + public string TestCollection => this._testCollections[0]; + + /// that can be used to manage the collections in Azure CosmosDB MongoDB. + public IMongoDatabase MongoDatabase { get; } + + /// Gets the manually created vector store record definition for Azure CosmosDB MongoDB test model. + public VectorStoreRecordDefinition HotelVectorStoreRecordDefinition { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + public AzureCosmosDBMongoDBVectorStoreFixture() + { + var configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile( + path: "testsettings.development.json", + optional: false, + reloadOnChange: true + ) + .AddEnvironmentVariables() + .Build(); + + var connectionString = GetConnectionString(configuration); + var client = new MongoClient(connectionString); + + this.MongoDatabase = client.GetDatabase("test"); + + this.HotelVectorStoreRecordDefinition = new() + { + Properties = + [ + new VectorStoreRecordKeyProperty("HotelId", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)), + new VectorStoreRecordDataProperty("HotelCode", typeof(int)), + new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)) { StoragePropertyName = "parking_is_included" }, + new VectorStoreRecordDataProperty("HotelRating", typeof(float)), + new VectorStoreRecordDataProperty("Tags", typeof(List)), + new VectorStoreRecordDataProperty("Description", typeof(string)), + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory?)) { Dimensions = 4, IndexKind = IndexKind.IvfFlat, DistanceFunction = DistanceFunction.CosineDistance } + ] + }; + } + + public async Task InitializeAsync() + { + foreach (var collection in this._testCollections) + { + await this.MongoDatabase.CreateCollectionAsync(collection); + } + } + + public async Task DisposeAsync() + { + foreach (var collection in this._testCollections) + { + await this.MongoDatabase.DropCollectionAsync(collection); + } + } + +#pragma warning disable CS8618 + public record AzureCosmosDBMongoDBHotel() + { + /// The key of the record. + [VectorStoreRecordKey] + public string HotelId { get; init; } + + /// A string metadata field. + [VectorStoreRecordData] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData] + public string Description { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } + } +#pragma warning restore CS8618 + + #region private + + private static string GetConnectionString(IConfigurationRoot configuration) + { + var settingValue = configuration["AzureCosmosDBMongoDB:ConnectionString"]; + if (string.IsNullOrWhiteSpace(settingValue)) + { + throw new ArgumentNullException($"{settingValue} string is not configured"); + } + + return settingValue; + } + + #endregion +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..f8a6053c78fd --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs @@ -0,0 +1,392 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Microsoft.SemanticKernel.Data; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using Xunit; +using static SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB.AzureCosmosDBMongoDBVectorStoreFixture; + +namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; + +[Collection("AzureCosmosDBMongoDBVectorStoreCollection")] +public class AzureCosmosDBMongoDBVectorStoreRecordCollectionTests(AzureCosmosDBMongoDBVectorStoreFixture fixture) +{ + private const string? SkipReason = "Azure CosmosDB MongoDB cluster is required"; + + [Theory(Skip = SkipReason)] + [InlineData("sk-test-hotels", true)] + [InlineData("nonexistentcollection", false)] + public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) + { + // Arrange + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, collectionName); + + // Act + var actual = await sut.CollectionExistsAsync(); + + // Assert + Assert.Equal(expectedExists, actual); + } + + [Fact(Skip = SkipReason)] + public async Task ItCanCreateCollectionAsync() + { + // Arrange + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + // Act + await sut.CreateCollectionAsync(); + + // Assert + Assert.True(await sut.CollectionExistsAsync()); + } + + [Theory(Skip = SkipReason)] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ItCanCreateCollectionUpsertAndGetAsync(bool includeVectors, bool useRecordDefinition) + { + // Arrange + const string HotelId = "55555555-5555-5555-5555-555555555555"; + + var collectionNamePostfix = useRecordDefinition ? "with-definition" : "with-type"; + var collectionName = $"collection-{collectionNamePostfix}"; + + var options = new AzureCosmosDBMongoDBVectorStoreRecordCollectionOptions + { + VectorStoreRecordDefinition = useRecordDefinition ? fixture.HotelVectorStoreRecordDefinition : null + }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, collectionName); + + var record = this.CreateTestHotel(HotelId); + + // Act + await sut.CreateCollectionAsync(); + var upsertResult = await sut.UpsertAsync(record); + var getResult = await sut.GetAsync(HotelId, new() { IncludeVectors = includeVectors }); + + // Assert + Assert.True(await sut.CollectionExistsAsync()); + await sut.DeleteCollectionAsync(); + + Assert.Equal(HotelId, upsertResult); + Assert.NotNull(getResult); + + Assert.Equal(record.HotelId, getResult.HotelId); + Assert.Equal(record.HotelName, getResult.HotelName); + Assert.Equal(record.HotelCode, getResult.HotelCode); + Assert.Equal(record.HotelRating, getResult.HotelRating); + Assert.Equal(record.ParkingIncluded, getResult.ParkingIncluded); + Assert.Equal(record.Tags.ToArray(), getResult.Tags.ToArray()); + Assert.Equal(record.Description, getResult.Description); + + if (includeVectors) + { + Assert.NotNull(getResult.DescriptionEmbedding); + Assert.Equal(record.DescriptionEmbedding!.Value.ToArray(), getResult.DescriptionEmbedding.Value.ToArray()); + } + else + { + Assert.Null(getResult.DescriptionEmbedding); + } + } + + [Fact(Skip = SkipReason)] + public async Task ItCanDeleteCollectionAsync() + { + // Arrange + const string TempCollectionName = "temp-test"; + await fixture.MongoDatabase.CreateCollectionAsync(TempCollectionName); + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, TempCollectionName); + + Assert.True(await sut.CollectionExistsAsync()); + + // Act + await sut.DeleteCollectionAsync(); + + // Assert + Assert.False(await sut.CollectionExistsAsync()); + } + + [Fact(Skip = SkipReason)] + public async Task ItCanGetAndDeleteRecordAsync() + { + // Arrange + const string HotelId = "55555555-5555-5555-5555-555555555555"; + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + var record = this.CreateTestHotel(HotelId); + + var upsertResult = await sut.UpsertAsync(record); + var getResult = await sut.GetAsync(HotelId); + + Assert.Equal(HotelId, upsertResult); + Assert.NotNull(getResult); + + // Act + await sut.DeleteAsync(HotelId); + + getResult = await sut.GetAsync(HotelId); + + // Assert + Assert.Null(getResult); + } + + [Fact(Skip = SkipReason)] + public async Task ItCanGetAndDeleteBatchAsync() + { + // Arrange + const string HotelId1 = "11111111-1111-1111-1111-111111111111"; + const string HotelId2 = "22222222-2222-2222-2222-222222222222"; + const string HotelId3 = "33333333-3333-3333-3333-333333333333"; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + var record1 = this.CreateTestHotel(HotelId1); + var record2 = this.CreateTestHotel(HotelId2); + var record3 = this.CreateTestHotel(HotelId3); + + var upsertResults = await sut.UpsertBatchAsync([record1, record2, record3]).ToListAsync(); + var getResults = await sut.GetBatchAsync([HotelId1, HotelId2, HotelId3]).ToListAsync(); + + Assert.Equal([HotelId1, HotelId2, HotelId3], upsertResults); + + Assert.NotNull(getResults.First(l => l.HotelId == HotelId1)); + Assert.NotNull(getResults.First(l => l.HotelId == HotelId2)); + Assert.NotNull(getResults.First(l => l.HotelId == HotelId3)); + + // Act + await sut.DeleteBatchAsync([HotelId1, HotelId2, HotelId3]); + + getResults = await sut.GetBatchAsync([HotelId1, HotelId2, HotelId3]).ToListAsync(); + + // Assert + Assert.Empty(getResults); + } + + [Fact(Skip = SkipReason)] + public async Task ItCanUpsertRecordAsync() + { + // Arrange + const string HotelId = "55555555-5555-5555-5555-555555555555"; + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + var record = this.CreateTestHotel(HotelId); + + var upsertResult = await sut.UpsertAsync(record); + var getResult = await sut.GetAsync(HotelId); + + Assert.Equal(HotelId, upsertResult); + Assert.NotNull(getResult); + + // Act + record.HotelName = "Updated name"; + record.HotelRating = 10; + + upsertResult = await sut.UpsertAsync(record); + getResult = await sut.GetAsync(HotelId); + + // Assert + Assert.NotNull(getResult); + Assert.Equal("Updated name", getResult.HotelName); + Assert.Equal(10, getResult.HotelRating); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertWithModelWorksCorrectlyAsync() + { + // Arrange + var definition = new VectorStoreRecordDefinition + { + Properties = new List + { + new VectorStoreRecordKeyProperty("Id", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)) + } + }; + + var model = new TestModel { Id = "key", HotelName = "Test Name" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + fixture.MongoDatabase, + fixture.TestCollection, + new() { VectorStoreRecordDefinition = definition }); + + // Act + var upsertResult = await sut.UpsertAsync(model); + var getResult = await sut.GetAsync(model.Id); + + // Assert + Assert.Equal("key", upsertResult); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertWithVectorStoreModelWorksCorrectlyAsync() + { + // Arrange + var model = new VectorStoreTestModel { Id = "key", HotelName = "Test Name" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + // Act + var upsertResult = await sut.UpsertAsync(model); + var getResult = await sut.GetAsync(model.Id); + + // Assert + Assert.Equal("key", upsertResult); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertWithBsonModelWorksCorrectlyAsync() + { + // Arrange + var definition = new VectorStoreRecordDefinition + { + Properties = new List + { + new VectorStoreRecordKeyProperty("Id", typeof(string)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)) + } + }; + + var model = new BsonTestModel { Id = "key", HotelName = "Test Name" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + fixture.MongoDatabase, + fixture.TestCollection, + new() { VectorStoreRecordDefinition = definition }); + + // Act + var upsertResult = await sut.UpsertAsync(model); + var getResult = await sut.GetAsync(model.Id); + + // Assert + Assert.Equal("key", upsertResult); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertWithBsonVectorStoreModelWorksCorrectlyAsync() + { + // Arrange + var model = new BsonVectorStoreTestModel { Id = "key", HotelName = "Test Name" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + // Act + var upsertResult = await sut.UpsertAsync(model); + var getResult = await sut.GetAsync(model.Id); + + // Assert + Assert.Equal("key", upsertResult); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + + [Fact(Skip = SkipReason)] + public async Task UpsertWithBsonVectorStoreWithNameModelWorksCorrectlyAsync() + { + // Arrange + var model = new BsonVectorStoreWithNameTestModel { Id = "key", HotelName = "Test Name" }; + + var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection(fixture.MongoDatabase, fixture.TestCollection); + + // Act + var upsertResult = await sut.UpsertAsync(model); + var getResult = await sut.GetAsync(model.Id); + + // Assert + Assert.Equal("key", upsertResult); + + Assert.NotNull(getResult); + Assert.Equal("key", getResult.Id); + Assert.Equal("Test Name", getResult.HotelName); + } + + #region private + + private AzureCosmosDBMongoDBHotel CreateTestHotel(string hotelId) + { + return new AzureCosmosDBMongoDBHotel + { + HotelId = hotelId, + HotelName = $"My Hotel {hotelId}", + HotelCode = 42, + HotelRating = 4.5f, + ParkingIncluded = true, + Tags = { "t1", "t2" }, + Description = "This is a great hotel.", + DescriptionEmbedding = new[] { 30f, 31f, 32f, 33f }, + }; + } + + private sealed class TestModel + { + public string? Id { get; set; } + + public string? HotelName { get; set; } + } + + private sealed class VectorStoreTestModel + { + [VectorStoreRecordKey] + public string? Id { get; set; } + + [VectorStoreRecordData(StoragePropertyName = "hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonTestModel + { + [BsonId] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreTestModel + { + [BsonId] + [VectorStoreRecordKey] + public string? Id { get; set; } + + [BsonElement("hotel_name")] + [VectorStoreRecordData] + public string? HotelName { get; set; } + } + + private sealed class BsonVectorStoreWithNameTestModel + { + [BsonId] + [VectorStoreRecordKey] + public string? Id { get; set; } + + [BsonElement("bson_hotel_name")] + [VectorStoreRecordData(StoragePropertyName = "storage_hotel_name")] + public string? HotelName { get; set; } + } + + #endregion +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs new file mode 100644 index 000000000000..9be1378b7b86 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; + +[Collection("AzureCosmosDBMongoDBVectorStoreCollection")] +public class AzureCosmosDBMongoDBVectorStoreTests(AzureCosmosDBMongoDBVectorStoreFixture fixture) +{ + private const string? SkipReason = "Azure CosmosDB MongoDB cluster is required"; + + [Fact(Skip = SkipReason)] + public async Task ItCanGetAListOfExistingCollectionNamesAsync() + { + // Arrange + var sut = new AzureCosmosDBMongoDBVectorStore(fixture.MongoDatabase); + + // Act + var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Contains("sk-test-hotels", collectionNames); + Assert.Contains("sk-test-contacts", collectionNames); + Assert.Contains("sk-test-addresses", collectionNames); + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLMemoryStoreTestsFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLMemoryStoreTestsFixture.cs index 1df46166e63f..7e6f376a8684 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLMemoryStoreTestsFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLMemoryStoreTestsFixture.cs @@ -47,7 +47,7 @@ public Task DisposeAsync() private static string GetSetting(IConfigurationRoot configuration, string settingName) { - var settingValue = configuration[$"AzureCosmosDB:{settingName}"]; + var settingValue = configuration[$"AzureCosmosDBNoSQL:{settingName}"]; if (string.IsNullOrWhiteSpace(settingValue)) { throw new ArgumentNullException($"{settingValue} string is not configured"); diff --git a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs index 2a4642a239b4..fc6a7ced0b89 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Data/RecordDefinition/IndexKind.cs @@ -34,6 +34,12 @@ public static class IndexKind /// public const string Flat = nameof(Flat); + /// + /// Inverted File with Flat Compression. Designed to enhance search efficiency by narrowing the search area through the use of neighbor partitions or clusters. + /// Also referred to as approximate nearest neighbor (ANN) search. + /// + public const string IvfFlat = nameof(IvfFlat); + /// /// Disk-based Approximate Nearest Neighbor algorithm designed for efficiently searching for approximate nearest neighbors (ANN) in high-dimensional spaces. /// The primary focus of DiskANN is to handle large-scale datasets that cannot fit entirely into memory, leveraging disk storage to store the data while maintaining fast search times.