Merge branch 'main' into feature-processes

README.md

@@ -111,7 +111,7 @@ on our Learn site. Each sample comes with a completed C# and Python project that
 
 1. 📖 [Getting Started](https://learn.microsoft.com/en-us/semantic-kernel/get-started/quick-start-guide)
 1. 🔌 [Detailed Samples](https://learn.microsoft.com/en-us/semantic-kernel/get-started/detailed-samples)
-1. 💡 [Concepts](https://learn.microsoft.com/en-us/semantic-kernel/concepts/agents)
+1. 💡 [Concepts](https://learn.microsoft.com/en-us/semantic-kernel/concepts/kernel)
 
 Finally, refer to our API references for more details on the C# and Python APIs:
 
@@ -136,7 +136,7 @@ in a different direction, but also to consider the impact on the larger ecosyste
 To learn more and get started:
 
 - Read the [documentation](https://aka.ms/sk/learn)
-- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/get-started/contributing) to the project
+- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/support/contributing) to the project
 - Ask questions in the [GitHub discussions](https://github.com/microsoft/semantic-kernel/discussions)
 - Ask questions in the [Discord community](https://aka.ms/SKDiscord)
 

docs/decisions/0034-rag-in-sk.md

@@ -143,7 +143,7 @@ var result = await kernel.InvokePromptAsync("{{budgetByYear}} What is my budget
 This approach is similar to Option 1, but data search step is part of prompt rendering process. Following list contains possible plugins to use for data search:
 
 - [ChatGPT Retrieval Plugin](https://github.com/openai/chatgpt-retrieval-plugin) - this plugin should be hosted as a separate service. It has integration with various [vector databases](https://github.com/openai/chatgpt-retrieval-plugin?tab=readme-ov-file#choosing-a-vector-database).
-- [SemanticKernel.Plugins.Memory.TextMemoryPlugin](https://www.nuget.org/packages/Microsoft.SemanticKernel.Plugins.Memory) - Semantic Kernel solution, which supports various [vector databases](https://learn.microsoft.com/en-us/semantic-kernel/memories/vector-db#available-connectors-to-vector-databases).
+- [SemanticKernel.Plugins.Memory.TextMemoryPlugin](https://www.nuget.org/packages/Microsoft.SemanticKernel.Plugins.Memory) - Semantic Kernel solution, which supports various vector databases.
 - Custom user plugin.
 
 ChatGPT Retrieval Plugin:

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBGenericDataModelMapperTests.cs

@@ -0,0 +1,310 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB;
+using Microsoft.SemanticKernel.Data;
+using MongoDB.Bson;
+using Xunit;
+
+namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests;
+
+/// <summary>
+/// Unit tests for <see cref="AzureCosmosDBMongoDBGenericDataModelMapper"/> class.
+/// </summary>
+public sealed class AzureCosmosDBMongoDBGenericDataModelMapperTests
+{
+    private static readonly VectorStoreRecordDefinition s_vectorStoreRecordDefinition = new()
+    {
+        Properties = new List<VectorStoreRecordProperty>
+        {
+            new VectorStoreRecordKeyProperty("Key", typeof(string)),
+            new VectorStoreRecordDataProperty("BoolDataProp", typeof(bool)),
+            new VectorStoreRecordDataProperty("NullableBoolDataProp", typeof(bool?)),
+            new VectorStoreRecordDataProperty("StringDataProp", typeof(string)),
+            new VectorStoreRecordDataProperty("IntDataProp", typeof(int)),
+            new VectorStoreRecordDataProperty("NullableIntDataProp", typeof(int?)),
+            new VectorStoreRecordDataProperty("LongDataProp", typeof(long)),
+            new VectorStoreRecordDataProperty("NullableLongDataProp", typeof(long?)),
+            new VectorStoreRecordDataProperty("FloatDataProp", typeof(float)),
+            new VectorStoreRecordDataProperty("NullableFloatDataProp", typeof(float?)),
+            new VectorStoreRecordDataProperty("DoubleDataProp", typeof(double)),
+            new VectorStoreRecordDataProperty("NullableDoubleDataProp", typeof(double?)),
+            new VectorStoreRecordDataProperty("DecimalDataProp", typeof(decimal)),
+            new VectorStoreRecordDataProperty("NullableDecimalDataProp", typeof(decimal?)),
+            new VectorStoreRecordDataProperty("DateTimeDataProp", typeof(DateTime)),
+            new VectorStoreRecordDataProperty("NullableDateTimeDataProp", typeof(DateTime?)),
+            new VectorStoreRecordDataProperty("TagListDataProp", typeof(List<string>)),
+            new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory<float>)),
+            new VectorStoreRecordVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory<float>?)),
+            new VectorStoreRecordVectorProperty("DoubleVector", typeof(ReadOnlyMemory<double>)),
+            new VectorStoreRecordVectorProperty("NullableDoubleVector", typeof(ReadOnlyMemory<double>?)),
+        },
+    };
+
+    private static readonly float[] s_floatVector = [1.0f, 2.0f, 3.0f];
+    private static readonly double[] s_doubleVector = [1.0f, 2.0f, 3.0f];
+    private static readonly List<string> s_taglist = ["tag1", "tag2"];
+
+    [Fact]
+    public void MapFromDataToStorageModelMapsAllSupportedTypes()
+    {
+        // Arrange
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(s_vectorStoreRecordDefinition);
+        var dataModel = new VectorStoreGenericDataModel<string>("key")
+        {
+            Data =
+            {
+                ["BoolDataProp"] = true,
+                ["NullableBoolDataProp"] = false,
+                ["StringDataProp"] = "string",
+                ["IntDataProp"] = 1,
+                ["NullableIntDataProp"] = 2,
+                ["LongDataProp"] = 3L,
+                ["NullableLongDataProp"] = 4L,
+                ["FloatDataProp"] = 5.0f,
+                ["NullableFloatDataProp"] = 6.0f,
+                ["DoubleDataProp"] = 7.0,
+                ["NullableDoubleDataProp"] = 8.0,
+                ["DecimalDataProp"] = 9.0m,
+                ["NullableDecimalDataProp"] = 10.0m,
+                ["DateTimeDataProp"] = new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(),
+                ["NullableDateTimeDataProp"] = new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(),
+                ["TagListDataProp"] = s_taglist,
+            },
+            Vectors =
+            {
+                ["FloatVector"] = new ReadOnlyMemory<float>(s_floatVector),
+                ["NullableFloatVector"] = new ReadOnlyMemory<float>(s_floatVector),
+                ["DoubleVector"] = new ReadOnlyMemory<double>(s_doubleVector),
+                ["NullableDoubleVector"] = new ReadOnlyMemory<double>(s_doubleVector),
+            },
+        };
+
+        // Act
+        var storageModel = sut.MapFromDataToStorageModel(dataModel);
+
+        // Assert
+        Assert.Equal("key", storageModel["_id"]);
+        Assert.Equal(true, (bool?)storageModel["BoolDataProp"]);
+        Assert.Equal(false, (bool?)storageModel["NullableBoolDataProp"]);
+        Assert.Equal("string", (string?)storageModel["StringDataProp"]);
+        Assert.Equal(1, (int?)storageModel["IntDataProp"]);
+        Assert.Equal(2, (int?)storageModel["NullableIntDataProp"]);
+        Assert.Equal(3L, (long?)storageModel["LongDataProp"]);
+        Assert.Equal(4L, (long?)storageModel["NullableLongDataProp"]);
+        Assert.Equal(5.0f, (float?)storageModel["FloatDataProp"].AsDouble);
+        Assert.Equal(6.0f, (float?)storageModel["NullableFloatDataProp"].AsNullableDouble);
+        Assert.Equal(7.0, (double?)storageModel["DoubleDataProp"]);
+        Assert.Equal(8.0, (double?)storageModel["NullableDoubleDataProp"]);
+        Assert.Equal(9.0m, (decimal?)storageModel["DecimalDataProp"]);
+        Assert.Equal(10.0m, (decimal?)storageModel["NullableDecimalDataProp"]);
+        Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(), storageModel["DateTimeDataProp"].ToUniversalTime());
+        Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(), storageModel["NullableDateTimeDataProp"].ToUniversalTime());
+        Assert.Equal(s_taglist, storageModel["TagListDataProp"]!.AsBsonArray.Select(x => (string)x!).ToArray());
+        Assert.Equal(s_floatVector, storageModel["FloatVector"]!.AsBsonArray.Select(x => (float)x.AsDouble!).ToArray());
+        Assert.Equal(s_floatVector, storageModel["NullableFloatVector"]!.AsBsonArray.Select(x => (float)x.AsNullableDouble!).ToArray());
+        Assert.Equal(s_doubleVector, storageModel["DoubleVector"]!.AsBsonArray.Select(x => (double)x!).ToArray());
+        Assert.Equal(s_doubleVector, storageModel["NullableDoubleVector"]!.AsBsonArray.Select(x => (double)x!).ToArray());
+    }
+
+    [Fact]
+    public void MapFromDataToStorageModelMapsNullValues()
+    {
+        // Arrange
+        VectorStoreRecordDefinition vectorStoreRecordDefinition = new()
+        {
+            Properties = new List<VectorStoreRecordProperty>
+            {
+                new VectorStoreRecordKeyProperty("Key", typeof(string)),
+                new VectorStoreRecordDataProperty("StringDataProp", typeof(string)),
+                new VectorStoreRecordDataProperty("NullableIntDataProp", typeof(int?)),
+                new VectorStoreRecordVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory<float>?)),
+            },
+        };
+
+        var dataModel = new VectorStoreGenericDataModel<string>("key")
+        {
+            Data =
+            {
+                ["StringDataProp"] = null,
+                ["NullableIntDataProp"] = null,
+            },
+            Vectors =
+            {
+                ["NullableFloatVector"] = null,
+            },
+        };
+
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(vectorStoreRecordDefinition);
+
+        // Act
+        var storageModel = sut.MapFromDataToStorageModel(dataModel);
+
+        // Assert
+        Assert.Equal(BsonNull.Value, storageModel["StringDataProp"]);
+        Assert.Equal(BsonNull.Value, storageModel["NullableIntDataProp"]);
+        Assert.Empty(storageModel["NullableFloatVector"].AsBsonArray);
+    }
+
+    [Fact]
+    public void MapFromStorageToDataModelMapsAllSupportedTypes()
+    {
+        // Arrange
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(s_vectorStoreRecordDefinition);
+        var storageModel = new BsonDocument
+        {
+            ["_id"] = "key",
+            ["BoolDataProp"] = true,
+            ["NullableBoolDataProp"] = false,
+            ["StringDataProp"] = "string",
+            ["IntDataProp"] = 1,
+            ["NullableIntDataProp"] = 2,
+            ["LongDataProp"] = 3L,
+            ["NullableLongDataProp"] = 4L,
+            ["FloatDataProp"] = 5.0f,
+            ["NullableFloatDataProp"] = 6.0f,
+            ["DoubleDataProp"] = 7.0,
+            ["NullableDoubleDataProp"] = 8.0,
+            ["DecimalDataProp"] = 9.0m,
+            ["NullableDecimalDataProp"] = 10.0m,
+            ["DateTimeDataProp"] = new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(),
+            ["NullableDateTimeDataProp"] = new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(),
+            ["TagListDataProp"] = BsonArray.Create(s_taglist),
+            ["FloatVector"] = BsonArray.Create(s_floatVector),
+            ["NullableFloatVector"] = BsonArray.Create(s_floatVector),
+            ["DoubleVector"] = BsonArray.Create(s_doubleVector),
+            ["NullableDoubleVector"] = BsonArray.Create(s_doubleVector)
+        };
+
+        // Act
+        var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true });
+
+        // Assert
+        Assert.Equal("key", dataModel.Key);
+        Assert.Equal(true, dataModel.Data["BoolDataProp"]);
+        Assert.Equal(false, dataModel.Data["NullableBoolDataProp"]);
+        Assert.Equal("string", dataModel.Data["StringDataProp"]);
+        Assert.Equal(1, dataModel.Data["IntDataProp"]);
+        Assert.Equal(2, dataModel.Data["NullableIntDataProp"]);
+        Assert.Equal(3L, dataModel.Data["LongDataProp"]);
+        Assert.Equal(4L, dataModel.Data["NullableLongDataProp"]);
+        Assert.Equal(5.0f, dataModel.Data["FloatDataProp"]);
+        Assert.Equal(6.0f, dataModel.Data["NullableFloatDataProp"]);
+        Assert.Equal(7.0, dataModel.Data["DoubleDataProp"]);
+        Assert.Equal(8.0, dataModel.Data["NullableDoubleDataProp"]);
+        Assert.Equal(9.0m, dataModel.Data["DecimalDataProp"]);
+        Assert.Equal(10.0m, dataModel.Data["NullableDecimalDataProp"]);
+        Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(), dataModel.Data["DateTimeDataProp"]);
+        Assert.Equal(new DateTime(2021, 1, 1, 0, 0, 0).ToUniversalTime(), dataModel.Data["NullableDateTimeDataProp"]);
+        Assert.Equal(s_taglist, dataModel.Data["TagListDataProp"]);
+        Assert.Equal(s_floatVector, ((ReadOnlyMemory<float>)dataModel.Vectors["FloatVector"]!).ToArray());
+        Assert.Equal(s_floatVector, ((ReadOnlyMemory<float>)dataModel.Vectors["NullableFloatVector"]!)!.ToArray());
+        Assert.Equal(s_doubleVector, ((ReadOnlyMemory<double>)dataModel.Vectors["DoubleVector"]!).ToArray());
+        Assert.Equal(s_doubleVector, ((ReadOnlyMemory<double>)dataModel.Vectors["NullableDoubleVector"]!)!.ToArray());
+    }
+
+    [Fact]
+    public void MapFromStorageToDataModelMapsNullValues()
+    {
+        // Arrange
+        VectorStoreRecordDefinition vectorStoreRecordDefinition = new()
+        {
+            Properties = new List<VectorStoreRecordProperty>
+            {
+                new VectorStoreRecordKeyProperty("Key", typeof(string)),
+                new VectorStoreRecordDataProperty("StringDataProp", typeof(string)),
+                new VectorStoreRecordDataProperty("NullableIntDataProp", typeof(int?)),
+                new VectorStoreRecordVectorProperty("NullableFloatVector", typeof(ReadOnlyMemory<float>?)),
+            },
+        };
+
+        var storageModel = new BsonDocument
+        {
+            ["_id"] = "key",
+            ["StringDataProp"] = BsonNull.Value,
+            ["NullableIntDataProp"] = BsonNull.Value,
+            ["NullableFloatVector"] = BsonNull.Value
+        };
+
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(vectorStoreRecordDefinition);
+
+        // Act
+        var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true });
+
+        // Assert
+        Assert.Equal("key", dataModel.Key);
+        Assert.Null(dataModel.Data["StringDataProp"]);
+        Assert.Null(dataModel.Data["NullableIntDataProp"]);
+        Assert.Null(dataModel.Vectors["NullableFloatVector"]);
+    }
+
+    [Fact]
+    public void MapFromStorageToDataModelThrowsForMissingKey()
+    {
+        // Arrange
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(s_vectorStoreRecordDefinition);
+        var storageModel = new BsonDocument();
+
+        // Act & Assert
+        var exception = Assert.Throws<VectorStoreRecordMappingException>(
+            () => sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true }));
+    }
+
+    [Fact]
+    public void MapFromDataToStorageModelSkipsMissingProperties()
+    {
+        // Arrange
+        VectorStoreRecordDefinition vectorStoreRecordDefinition = new()
+        {
+            Properties = new List<VectorStoreRecordProperty>
+            {
+                new VectorStoreRecordKeyProperty("Key", typeof(string)),
+                new VectorStoreRecordDataProperty("StringDataProp", typeof(string)),
+                new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory<float>)),
+            },
+        };
+
+        var dataModel = new VectorStoreGenericDataModel<string>("key");
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(vectorStoreRecordDefinition);
+
+        // Act
+        var storageModel = sut.MapFromDataToStorageModel(dataModel);
+
+        // Assert
+        Assert.Equal("key", (string?)storageModel["_id"]);
+        Assert.False(storageModel.Contains("StringDataProp"));
+        Assert.False(storageModel.Contains("FloatVector"));
+    }
+
+    [Fact]
+    public void MapFromStorageToDataModelSkipsMissingProperties()
+    {
+        // Arrange
+        VectorStoreRecordDefinition vectorStoreRecordDefinition = new()
+        {
+            Properties = new List<VectorStoreRecordProperty>
+            {
+                new VectorStoreRecordKeyProperty("Key", typeof(string)),
+                new VectorStoreRecordDataProperty("StringDataProp", typeof(string)),
+                new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory<float>)),
+            },
+        };
+
+        var storageModel = new BsonDocument
+        {
+            ["_id"] = "key"
+        };
+
+        var sut = new AzureCosmosDBMongoDBGenericDataModelMapper(vectorStoreRecordDefinition);
+
+        // Act
+        var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true });
+
+        // Assert
+        Assert.Equal("key", dataModel.Key);
+        Assert.False(dataModel.Data.ContainsKey("StringDataProp"));
+        Assert.False(dataModel.Vectors.ContainsKey("FloatVector"));
+    }
+}

dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchGenericDataModelMapper.cs

@@ -11,7 +11,7 @@
 namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
 
 /// <summary>
-/// A mapper that maps between the generic semantic kernel data model and the model that the data is stored in in Azure AI Search.
+/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within Azure AI Search.
 /// </summary>
 internal class AzureAISearchGenericDataModelMapper : IVectorStoreRecordMapper<VectorStoreGenericDataModel<string>, JsonObject>
 {

dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBConstants.cs

@@ -1,5 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
+using System.Collections.Generic;
+
 namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB;
 
 /// <summary>
@@ -12,4 +15,39 @@ internal static class AzureCosmosDBMongoDBConstants
 
     /// <summary>Reserved key property name in data model.</summary>
     internal const string DataModelReservedKeyPropertyName = "Id";
+
+    /// <summary>A <see cref="HashSet{Type}"/> containing the supported key types.</summary>
+    internal static readonly HashSet<Type> SupportedKeyTypes =
+    [
+        typeof(string)
+    ];
+
+    /// <summary>A <see cref="HashSet{Type}"/> containing the supported data property types.</summary>
+    internal static readonly HashSet<Type> SupportedDataTypes =
+    [
+        typeof(bool),
+        typeof(bool?),
+        typeof(string),
+        typeof(int),
+        typeof(int?),
+        typeof(long),
+        typeof(long?),
+        typeof(float),
+        typeof(float?),
+        typeof(double),
+        typeof(double?),
+        typeof(decimal),
+        typeof(decimal?),
+        typeof(DateTime),
+        typeof(DateTime?),
+    ];
+
+    /// <summary>A <see cref="HashSet{Type}"/> containing the supported vector types.</summary>
+    internal static readonly HashSet<Type> SupportedVectorTypes =
+    [
+        typeof(ReadOnlyMemory<float>),
+        typeof(ReadOnlyMemory<float>?),
+        typeof(ReadOnlyMemory<double>),
+        typeof(ReadOnlyMemory<double>?)
+    ];
 }