Skip to content

Commit

Permalink
.Net: Fix bug where filtered tag field throws. (#9804)
Browse files Browse the repository at this point in the history
### Motivation and Context

#9799

### Description

Missed the mapping for enumerable strings for index creation, so needed
to add it, and a proper check for unsupported types.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄
  • Loading branch information
westey-m authored Nov 25, 2024
1 parent ada7ba6 commit b8680fa
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,20 @@ await this.RunOperationAsync(
foreach (var dataProperty in dataProperties)
{
var storageFieldName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName);
var schemaType = QdrantVectorStoreCollectionCreateMapping.s_schemaTypeMap[dataProperty.PropertyType!];

if (QdrantVectorStoreCollectionCreateMapping.s_schemaTypeMap.TryGetValue(dataProperty.PropertyType!, out PayloadSchemaType schemaType))
{
// Do nothing since schemaType is already set.
}
else if (VectorStoreRecordPropertyVerification.IsSupportedEnumerableType(dataProperty.PropertyType) && VectorStoreRecordPropertyVerification.GetCollectionElementType(dataProperty.PropertyType) == typeof(string))
{
// For enumerable of strings, use keyword schema type, since this allows tag filtering.
schemaType = PayloadSchemaType.Keyword;
}
else
{
throw new InvalidOperationException($"Property {nameof(VectorStoreRecordDataProperty.IsFilterable)} on {nameof(VectorStoreRecordDataProperty)} '{dataProperty.DataModelPropertyName}' is set to true, but the property type is not supported for filtering. The Qdrant VectorStore supports filtering on {string.Join(", ", QdrantVectorStoreCollectionCreateMapping.s_schemaTypeMap.Keys.Select(x => x.Name))} properties only.");
}

await this.RunOperationAsync(
"CreatePayloadIndex",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public QdrantVectorStoreFixture()
new VectorStoreRecordDataProperty("HotelCode", typeof(int)) { IsFilterable = true },
new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)) { IsFilterable = true, StoragePropertyName = "parking_is_included" },
new VectorStoreRecordDataProperty("HotelRating", typeof(float)) { IsFilterable = true },
new VectorStoreRecordDataProperty("Tags", typeof(List<string>)),
new VectorStoreRecordDataProperty("Tags", typeof(List<string>)) { IsFilterable = true },
new VectorStoreRecordDataProperty("Description", typeof(string)),
new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory<float>?)) { Dimensions = VectorDimensions, DistanceFunction = DistanceFunction.ManhattanDistance }
}
Expand Down Expand Up @@ -146,11 +146,17 @@ await this.QdrantClient.CreateCollectionAsync(

// Create test data common to both named and unnamed vectors.
var tags = new ListValue();
tags.Values.Add("t1");
tags.Values.Add("t2");
tags.Values.Add("t11.1");
tags.Values.Add("t11.2");
var tagsValue = new Value();
tagsValue.ListValue = tags;

var tags2 = new ListValue();
tags2.Values.Add("t13.1");
tags2.Values.Add("t13.2");
var tagsValue2 = new Value();
tagsValue2.ListValue = tags2;

// Create some test data using named vectors.
var embedding = await this.EmbeddingGenerator.GenerateEmbeddingAsync("This is a great hotel.");
var embeddingArray = embedding.ToArray();
Expand Down Expand Up @@ -183,7 +189,7 @@ await this.QdrantClient.CreateCollectionAsync(
{
Id = 13,
Vectors = new Vectors { Vectors_ = namedVectors3 },
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Description"] = "This is a great hotel." }
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel." }
},
new PointStruct
{
Expand Down Expand Up @@ -214,7 +220,7 @@ await this.QdrantClient.CreateCollectionAsync(
{
Id = 13,
Vectors = embeddingArray,
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Description"] = "This is a great hotel." }
Payload = { ["HotelName"] = "My Hotel 13", ["HotelCode"] = 13, ["parking_is_included"] = false, ["Tags"] = tagsValue2, ["Description"] = "This is a great hotel." }
},
];

Expand Down Expand Up @@ -327,7 +333,7 @@ public record HotelInfo()
[VectorStoreRecordData(IsFilterable = true, StoragePropertyName = "parking_is_included")]
public bool ParkingIncluded { get; set; }

[VectorStoreRecordData]
[VectorStoreRecordData(IsFilterable = true)]
public List<string> Tags { get; set; } = new List<string>();

/// <summary>A data field.</summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool hasNamedVec
var vector = await fixture.EmbeddingGenerator.GenerateEmbeddingAsync("A great hotel");
var actual = await sut.VectorizedSearchAsync(
vector,
new VectorSearchOptions { Filter = new VectorSearchFilter().EqualTo("HotelCode", 30) });
new VectorSearchOptions { Filter = new VectorSearchFilter().EqualTo("HotelCode", 30).AnyTagEqualTo("Tags", "t2") });

// Assert
var collectionExistResult = await sut.CollectionExistsAsync();
Expand Down Expand Up @@ -221,8 +221,8 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool useRecordDefinition,
Assert.True(getResult?.ParkingIncluded);
Assert.Equal(4.5f, getResult?.HotelRating);
Assert.Equal(2, getResult?.Tags.Count);
Assert.Equal("t1", getResult?.Tags[0]);
Assert.Equal("t2", getResult?.Tags[1]);
Assert.Equal("t11.1", getResult?.Tags[0]);
Assert.Equal("t11.2", getResult?.Tags[1]);
Assert.Equal("This is a great hotel.", getResult?.Description);
if (withEmbeddings)
{
Expand Down Expand Up @@ -389,7 +389,7 @@ public async Task ItCanSearchWithFilterAsync(bool useRecordDefinition, string co

// Act.
var vector = await fixture.EmbeddingGenerator.GenerateEmbeddingAsync("A great hotel");
var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelName", "My Hotel 11") : new VectorSearchFilter().AnyTagEqualTo("Tags", "t1");
var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelName", "My Hotel 13") : new VectorSearchFilter().AnyTagEqualTo("Tags", "t13.2");
var actual = await sut.VectorizedSearchAsync(
vector,
new()
Expand All @@ -402,12 +402,11 @@ public async Task ItCanSearchWithFilterAsync(bool useRecordDefinition, string co
Assert.Single(searchResults);

var searchResultRecord = searchResults.First().Record;
Assert.Equal(11ul, searchResultRecord?.HotelId);
Assert.Equal("My Hotel 11", searchResultRecord?.HotelName);
Assert.Equal(11, searchResultRecord?.HotelCode);
Assert.Equal(4.5f, searchResultRecord?.HotelRating);
Assert.Equal(true, searchResultRecord?.ParkingIncluded);
Assert.Equal(new string[] { "t1", "t2" }, searchResultRecord?.Tags.ToArray());
Assert.Equal(13ul, searchResultRecord?.HotelId);
Assert.Equal("My Hotel 13", searchResultRecord?.HotelName);
Assert.Equal(13, searchResultRecord?.HotelCode);
Assert.Equal(false, searchResultRecord?.ParkingIncluded);
Assert.Equal(new string[] { "t13.1", "t13.2" }, searchResultRecord?.Tags.ToArray());
Assert.Equal("This is a great hotel.", searchResultRecord?.Description);
}

Expand Down Expand Up @@ -448,7 +447,7 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync()
Assert.Equal(11, baseSetGetResult.Data["HotelCode"]);
Assert.True((bool)baseSetGetResult.Data["ParkingIncluded"]!);
Assert.Equal(4.5f, baseSetGetResult.Data["HotelRating"]);
Assert.Equal(new[] { "t1", "t2" }, ((List<string>)baseSetGetResult.Data["Tags"]!).ToArray());
Assert.Equal(new[] { "t11.1", "t11.2" }, ((List<string>)baseSetGetResult.Data["Tags"]!).ToArray());
Assert.Equal("This is a great hotel.", baseSetGetResult.Data["Description"]);
Assert.NotNull(baseSetGetResult.Vectors["DescriptionEmbedding"]);
Assert.IsType<ReadOnlyMemory<float>>(baseSetGetResult.Vectors["DescriptionEmbedding"]);
Expand Down

0 comments on commit b8680fa

Please sign in to comment.