diff --git a/Directory.Packages.props b/Directory.Packages.props index 4f15290..6de1e75 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,7 +12,7 @@ - + diff --git a/MEVD/src/Chroma/Chroma.csproj b/MEVD/src/Chroma/Chroma.csproj index ca4a853..ae75898 100644 --- a/MEVD/src/Chroma/Chroma.csproj +++ b/MEVD/src/Chroma/Chroma.csproj @@ -1,7 +1,7 @@  - 1.0.1-preview + 1.0.2-preview CommunityToolkit.VectorData.Chroma $(AssemblyName) net10.0;net8.0;netstandard2.0;net462 diff --git a/MEVD/src/Chroma/ChromaCollection.cs b/MEVD/src/Chroma/ChromaCollection.cs index 2e19dbe..f9d2aba 100644 --- a/MEVD/src/Chroma/ChromaCollection.cs +++ b/MEVD/src/Chroma/ChromaCollection.cs @@ -408,9 +408,7 @@ public async IAsyncEnumerable> HybridSearchAsync { var index = await _records.FindBm25IndexAsync(textProperty.StorageName, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException( - $"The Chroma collection '{Name}' has no BM25 index on the text of the property '{textProperty.ModelName}', which hybrid search needs. " + - "Only Chroma Cloud has BM25 indexes."); + ?? throw new InvalidOperationException(NoBm25IndexMessage(textProperty)); return await _records.SearchAsync( new ChromaSearch { @@ -492,6 +490,27 @@ public override async IAsyncEnumerable GetAsync(Expression _chromaClient.Client.Options.IsChromaCloud + && _bm25Properties.Any(p => ChromaCollectionCreateMapping.FitsChromaCloud(ChromaCollectionCreateMapping.GetBm25IndexKey(p, _mapper.DocumentProperty))); + + // Why the collection has no BM25 index on the property: on Chroma Cloud, the key of the index can be beyond its quota, or + // the collection was created without the index. + private string NoBm25IndexMessage(DataPropertyModel textProperty) + { + var message = $"The Chroma collection '{Name}' has no BM25 index on the text of the property '{textProperty.ModelName}', which hybrid search needs. "; + if (!_chromaClient.Client.Options.IsChromaCloud) + { + return message + "Only Chroma Cloud has BM25 indexes."; + } + + var key = ChromaCollectionCreateMapping.GetBm25IndexKey(textProperty, _mapper.DocumentProperty); + return ChromaCollectionCreateMapping.FitsChromaCloud(key) + ? message + "The collection was created without it, for example by another client." + : message + $"Its key, '{key}', is beyond the {ChromaCloudQuotas.MaxMetadataKeyBytes} bytes that Chroma Cloud takes for a metadata key, so the collection was created without it. Use a shorter storage name."; + } + /// public override object? GetService(Type serviceType, object? serviceKey = null) { @@ -501,8 +520,7 @@ public override async IAsyncEnumerable GetAsync(Expression) ? (_bm25Properties.Count > 0 && _chromaClient.Client.Options.IsChromaCloud ? this : null) : + serviceType == typeof(IKeywordHybridSearchable) ? (HasBm25IndexChromaCloudTakes ? this : null) : serviceType.IsInstanceOfType(this) ? this : null; } diff --git a/MEVD/src/Chroma/ChromaCollectionCreateMapping.cs b/MEVD/src/Chroma/ChromaCollectionCreateMapping.cs index 74c229b..bbd12e2 100644 --- a/MEVD/src/Chroma/ChromaCollectionCreateMapping.cs +++ b/MEVD/src/Chroma/ChromaCollectionCreateMapping.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text; using ChromaDB.Client; using ChromaDB.Client.Models; using Microsoft.Extensions.VectorData; @@ -44,6 +45,23 @@ public static ChromaCollectionDefinition MapCollectionDefinition(string name, Ve public static List GetBm25Properties(CollectionModel model) => model.DataProperties.Where(ChromaFieldMapping.IsFullTextString).ToList(); + /// + /// Get the metadata key of the BM25 index of a property, as the client names it: document_bm25 for the property + /// stored as the document, else the storage name with _bm25. + /// + /// The full-text property. + /// The property stored as the document, or . + public static string GetBm25IndexKey(DataPropertyModel property, DataPropertyModel? documentProperty) + => property == documentProperty ? "document_bm25" : property.StorageName + "_bm25"; + + /// + /// Get a value indicating whether Chroma Cloud takes a metadata key. The client creates no BM25 index whose key is longer + /// than that. + /// + /// The metadata key. + public static bool FitsChromaCloud(string key) + => Encoding.UTF8.GetByteCount(key) <= ChromaCloudQuotas.MaxMetadataKeyBytes; + /// /// Get the Chroma distance function, called space, for the given . /// If none is configured, the default is cosine. diff --git a/MEVD/src/Chroma/ChromaModelBuilder.cs b/MEVD/src/Chroma/ChromaModelBuilder.cs index e1b4704..1280e29 100644 --- a/MEVD/src/Chroma/ChromaModelBuilder.cs +++ b/MEVD/src/Chroma/ChromaModelBuilder.cs @@ -52,6 +52,13 @@ protected override void ValidateProperty(PropertyModel propertyModel, VectorStor $"Property '{propertyModel.ModelName}' has the storage name '{propertyModel.StorageName}', but Chroma does not accept a metadata key that is empty or starts with # or $."); } + // Chroma drops the metadata keys that start with chroma: from every read: the property would always read back as null. + if (propertyModel is DataPropertyModel { StorageName: var storageName } && storageName.StartsWith("chroma:", StringComparison.Ordinal)) + { + throw new NotSupportedException( + $"Property '{propertyModel.ModelName}' has the storage name '{storageName}', but Chroma drops the metadata keys that start with chroma: from every read."); + } + switch (propertyModel) { case VectorPropertyModel vectorProperty: diff --git a/MEVD/src/Chroma/README.md b/MEVD/src/Chroma/README.md index 5826a50..a0a8f6b 100644 --- a/MEVD/src/Chroma/README.md +++ b/MEVD/src/Chroma/README.md @@ -26,10 +26,15 @@ For more information, see the [Microsoft.Extensions.VectorData documentation](ht - On Chroma Cloud, a vector search returns at most 300 results, `Skip` included, the default quota of Chroma Cloud: beyond that, Chroma Cloud answers with a quota error. - Chroma does not store empty lists, so an empty array or list comes back as `null`. +- A data property whose storage name is empty, or starts with `#`, `$` or `chroma:`, throws a `NotSupportedException` when the collection object is created. Chroma rejects the first ones in a write, and drops the `chroma:` keys from every read, so such a property would always come back as `null`. - Array and list properties, like the namespaces of the `TextSearchStore` of Semantic Kernel, need Chroma 1.5.0 or later. +- A collection created by another client without a space has the space l2. A search on it with a distance function of another space throws, except on Chroma 1.0.0 to 1.0.5: there the provider cannot see the space, and the search returns wrong scores without an error. Use Chroma 1.0.6 or later. - On self-hosted Chroma 1.0.21 to 1.5.9, a vector search can miss a record after an upsert, while a get still finds it. It is KD-49 in the [compatibility notes](https://github.com/ChromaDotNet/ChromaDB.Client/blob/main/docs/COMPATIBILITY.md#records-lost-after-an-update-of-their-embeddings) of ChromaDotNet.Client. The environment variable `RAYON_NUM_THREADS=1` on the server makes it much rarer. In the client, `ChromaUpsertStrategy.SkipUnchangedEmbeddings` writes a record whose vector does not change without its vector, so that write does not touch the vector index. +- On self-hosted Chroma 1.5.9, while upserts of the same record run at the same time, a read can return a mix of two of them: the keys that one sets to `null` and the other writes. Once the writes end, the record is one of them. +- On Chroma 1.5.9, with the cosine space and vectors that are nearly parallel, a vector search over all the records can miss some of them, and pages can repeat records. It comes from the approximate index of Chroma; with the l2 space the same records are all found. - On Chroma Cloud, a record has at most 32 metadata keys, and a null value counts as one. The provider writes every data property, null or not. A write with null values that would take a record beyond 32 keys throws an `ArgumentException`. With `WithReadsBeyondKeyQuota()`, the client reads such a record first and sends only the null values of the keys it has. Two writes of the same record at the same time can then mix. - On Chroma Cloud, a metadata key has at most 36 bytes in UTF-8. With a data property whose storage name is longer, the collection is created, but every upsert throws a `VectorStoreException` with the quota error of Chroma Cloud. +- With two or more full-text properties, each has a BM25 index of its own, whose key is its storage name with `_bm25`. On Chroma Cloud, a storage name of more than 31 bytes makes that key longer than 36 bytes: the collection is created without that index, and a hybrid search on that property throws an `InvalidOperationException` that says why. - On Chroma Cloud, `Contains` on the full-text property throws an `ArgumentException` for a text of fewer than 3 characters. Once Chroma Cloud has indexed the records, such a filter would find no record, and its negation every record. The connection string does not set `WithUpsertStrategy` and `WithReadsBeyondKeyQuota()`. Set them on the options of a `ChromaClient` created in code, and pass it to the vector store: diff --git a/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs b/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs index 27fede0..8485578 100644 --- a/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs +++ b/MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs @@ -120,6 +120,67 @@ public void RejectsAFullTextPropertyWhoseStorageNameStartsWithAHash() Assert.Throws(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition })); } + [Theory] + [InlineData("chroma:tag")] + [InlineData("chroma:document")] + [InlineData("chroma:uri")] + [InlineData("chroma:")] + public void RejectsADataPropertyWhoseStorageNameStartsWithTheChromaPrefix(string storageName) + { + // Chroma drops the metadata keys that start with chroma: from every read: the property would always read back as null. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Tag", typeof(string)) { StorageName = storageName }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4), + ] + }; + + var exception = Assert.Throws(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition })); + Assert.Contains($"'{storageName}'", exception.Message); + } + + [Fact] + public void RejectsAFullTextPropertyWhoseStorageNameStartsWithTheChromaPrefix() + { + // The full-text property stored as the document is also a metadata key: the client copies the document into it. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Text", typeof(string)) { StorageName = "chroma:document", IsFullTextIndexed = true }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4), + ] + }; + + Assert.Throws(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition })); + } + + [Theory] + [InlineData("Chroma:tag")] + [InlineData("CHROMA:tag")] + [InlineData("chroma_tag")] + [InlineData("tag:chroma:")] + public void AcceptsAStorageNameThatDoesNotStartWithTheChromaPrefix(string storageName) + { + // Chroma drops only the keys that start with chroma: in lower case. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Tag", typeof(string)) { StorageName = storageName }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4), + ] + }; + + using var sut = new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition }); + Assert.Equal(TestCollectionName, sut.Name); + } + [Theory] [InlineData("tag#1")] [InlineData("price$usd")] @@ -218,6 +279,21 @@ public void GetServiceOffersHybridSearchOnlyWithFullTextPropertiesOnChromaCloud( Assert.Null(noFullText.GetService(typeof(IKeywordHybridSearchable>))); } + [Fact] + public void GetServiceOffersHybridSearchOnlyWithABm25IndexKeyThatChromaCloudTakes() + { + // Arrange: the client leaves out on Chroma Cloud a BM25 index whose key is beyond 36 bytes. + this._chromaClientMock + .Setup(x => x.Options) + .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + using var oneFits = this.CreateCollection(); + using var noneFits = this.CreateCollection(); + + // Act and assert. + Assert.NotNull(oneFits.GetService(typeof(IKeywordHybridSearchable))); + Assert.Null(noneFits.GetService(typeof(IKeywordHybridSearchable))); + } + #endregion #region Get by key @@ -400,7 +476,22 @@ public async Task HybridSearchThrowsWithoutABm25IndexAsync() // Act and assert. var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync()); - Assert.Contains("Chroma Cloud", exception.Message); + Assert.Contains("Only Chroma Cloud has BM25 indexes.", exception.Message); + } + + [Fact] + public async Task HybridSearchOnChromaCloudSaysTheCollectionWasCreatedWithoutTheBm25IndexAsync() + { + // Arrange: Chroma Cloud, with a key that fits but no index, as in a collection that another client created. + this._chromaClientMock + .Setup(x => x.Options) + .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + using var sut = this.CreateHybridCollection(); + + // Act and assert. + var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync()); + Assert.Contains("created without it", exception.Message); + Assert.DoesNotContain("Only Chroma Cloud", exception.Message); } [Fact] @@ -414,6 +505,21 @@ public async Task HybridSearchThrowsWithoutAnIndexOnTheChosenPropertyAsync() Assert.Contains("'Review'", exception.Message); } + [Fact] + public async Task HybridSearchSaysWhyChromaCloudHasNoBm25IndexOnALongStorageNameAsync() + { + // Arrange: Chroma Cloud, with only the index of the property that fits. + this._chromaClientMock + .Setup(x => x.Options) + .Returns(new ChromaConfigurationOptions("https://api.trychroma.com")); + using var sut = this.CreateHybridCollection(Bm25Index("Review_with_a_long_name_xxxxxxx_bm25", "Review_with_a_long_name_xxxxxxx")); + + // Act and assert. + var exception = await Assert.ThrowsAsync(async () => await sut.HybridSearchAsync(new ReadOnlyMemory([1, 2, 3, 4]), ["pool"], top: 1, new() { AdditionalProperty = h => h.Description }, this._testCancellationToken).ToListAsync()); + Assert.Contains("'Description_with_a_long_name_xxx_bm25'", exception.Message); + Assert.Contains("36 bytes", exception.Message); + } + [Fact] public async Task HybridSearchCannotReadVectorsWithEmbeddingGenerationAsync() { diff --git a/MEVD/test/Chroma.UnitTests/ChromaHotel.cs b/MEVD/test/Chroma.UnitTests/ChromaHotel.cs index b0cbcb1..e78dc6c 100644 --- a/MEVD/test/Chroma.UnitTests/ChromaHotel.cs +++ b/MEVD/test/Chroma.UnitTests/ChromaHotel.cs @@ -102,6 +102,39 @@ public sealed class TwoFullTextHotel public ReadOnlyMemory? Embedding { get; set; } } +// Two full-text properties, each with a BM25 index of its own: the key of the first, Description_with_a_long_name_xxx_bm25, +// has 37 bytes, beyond the 36 of Chroma Cloud; the key of the second, Review_with_a_long_name_xxxxxxx_bm25, has 36. +public sealed class LongFullTextHotel +{ + [VectorStoreKey] + public string HotelId { get; set; } = default!; + + [VectorStoreData(IsFullTextIndexed = true, StorageName = "Description_with_a_long_name_xxx")] + public string? Description { get; set; } + + [VectorStoreData(IsFullTextIndexed = true, StorageName = "Review_with_a_long_name_xxxxxxx")] + public string? Review { get; set; } + + [VectorStoreVector(4)] + public ReadOnlyMemory? Embedding { get; set; } +} + +// As LongFullTextHotel, with both keys beyond the 36 bytes of Chroma Cloud. +public sealed class TooLongFullTextHotel +{ + [VectorStoreKey] + public string HotelId { get; set; } = default!; + + [VectorStoreData(IsFullTextIndexed = true, StorageName = "Description_with_a_long_name_xxx")] + public string? Description { get; set; } + + [VectorStoreData(IsFullTextIndexed = true, StorageName = "Review_with_a_long_name_xxxxxxxx")] + public string? Review { get; set; } + + [VectorStoreVector(4)] + public ReadOnlyMemory? Embedding { get; set; } +} + public sealed class FullTextHotel { [VectorStoreKey]