Chroma: update ChromaDotNet.Client to 2.18.0, reject chroma: storage names, explain missing BM25 indexes on Chroma Cloud

Directory.Packages.props

@@ -12,7 +12,7 @@
     <PackageVersion Include="Microsoft.Extensions.VectorData.ConformanceTests" Version="10.8.2" />
 
     <PackageVersion Include="Azure.Search.Documents" Version="11.7.0" />
-    <PackageVersion Include="ChromaDotNet.Client" Version="2.14.0" />
+    <PackageVersion Include="ChromaDotNet.Client" Version="2.18.0" />
     <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.61.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="6.1.6" />
     <PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.13" />

MEVD/src/Chroma/Chroma.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <Version>1.0.1-preview</Version>
+    <Version>1.0.2-preview</Version>
     <AssemblyName>CommunityToolkit.VectorData.Chroma</AssemblyName>
     <RootNamespace>$(AssemblyName)</RootNamespace>
     <TargetFrameworks>net10.0;net8.0;netstandard2.0;net462</TargetFrameworks>

MEVD/src/Chroma/ChromaCollection.cs

@@ -408,9 +408,7 @@ public async IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TIn
             async () =>
             {
                 var index = await _records.FindBm25IndexAsync(textProperty.StorageName, cancellationToken).ConfigureAwait(false)
-                    ?? throw new InvalidOperationException(
-                        $"The Chroma collection '{Name}' has no BM25 index on the text of the property '{textProperty.ModelName}', which hybrid search needs. " +
-                        "Only Chroma Cloud has BM25 indexes.");
+                    ?? throw new InvalidOperationException(NoBm25IndexMessage(textProperty));
                 return await _records.SearchAsync(
                     new ChromaSearch
                     {
@@ -492,6 +490,27 @@ public override async IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord
         }
     }
 
+    // Hybrid search needs the BM25 indexes, which the client creates only on Chroma Cloud, and only with a key it takes.
+    private bool HasBm25IndexChromaCloudTakes
+        => _chromaClient.Client.Options.IsChromaCloud
+            && _bm25Properties.Any(p => ChromaCollectionCreateMapping.FitsChromaCloud(ChromaCollectionCreateMapping.GetBm25IndexKey(p, _mapper.DocumentProperty)));
+
+    // Why the collection has no BM25 index on the property: on Chroma Cloud, the key of the index can be beyond its quota, or
+    // the collection was created without the index.
+    private string NoBm25IndexMessage(DataPropertyModel textProperty)
+    {
+        var message = $"The Chroma collection '{Name}' has no BM25 index on the text of the property '{textProperty.ModelName}', which hybrid search needs. ";
+        if (!_chromaClient.Client.Options.IsChromaCloud)
+        {
+            return message + "Only Chroma Cloud has BM25 indexes.";
+        }
+
+        var key = ChromaCollectionCreateMapping.GetBm25IndexKey(textProperty, _mapper.DocumentProperty);
+        return ChromaCollectionCreateMapping.FitsChromaCloud(key)
+            ? message + "The collection was created without it, for example by another client."
+            : message + $"Its key, '{key}', is beyond the {ChromaCloudQuotas.MaxMetadataKeyBytes} bytes that Chroma Cloud takes for a metadata key, so the collection was created without it. Use a shorter storage name.";
+    }
+
     /// <inheritdoc />
     public override object? GetService(Type serviceType, object? serviceKey = null)
     {
@@ -501,8 +520,7 @@ public override async IAsyncEnumerable<TRecord> GetAsync(Expression<Func<TRecord
             serviceKey is not null ? null :
             serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata :
             serviceType == typeof(ChromaClient) ? _chromaClient.Client :
-            // Hybrid search needs the BM25 indexes, which the client creates only on Chroma Cloud.
-            serviceType == typeof(IKeywordHybridSearchable<TRecord>) ? (_bm25Properties.Count > 0 && _chromaClient.Client.Options.IsChromaCloud ? this : null) :
+            serviceType == typeof(IKeywordHybridSearchable<TRecord>) ? (HasBm25IndexChromaCloudTakes ? this : null) :
             serviceType.IsInstanceOfType(this) ? this :
             null;
     }

MEVD/src/Chroma/ChromaCollectionCreateMapping.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
 using ChromaDB.Client;
 using ChromaDB.Client.Models;
 using Microsoft.Extensions.VectorData;
@@ -44,6 +45,23 @@ public static ChromaCollectionDefinition MapCollectionDefinition(string name, Ve
     public static List<DataPropertyModel> GetBm25Properties(CollectionModel model)
         => model.DataProperties.Where(ChromaFieldMapping.IsFullTextString).ToList();
 
+    /// <summary>
+    /// Get the metadata key of the BM25 index of a property, as the client names it: <c>document_bm25</c> for the property
+    /// stored as the document, else the storage name with <c>_bm25</c>.
+    /// </summary>
+    /// <param name="property">The full-text property.</param>
+    /// <param name="documentProperty">The property stored as the document, or <see langword="null"/>.</param>
+    public static string GetBm25IndexKey(DataPropertyModel property, DataPropertyModel? documentProperty)
+        => property == documentProperty ? "document_bm25" : property.StorageName + "_bm25";
+
+    /// <summary>
+    /// Get a value indicating whether Chroma Cloud takes a metadata key. The client creates no BM25 index whose key is longer
+    /// than that.
+    /// </summary>
+    /// <param name="key">The metadata key.</param>
+    public static bool FitsChromaCloud(string key)
+        => Encoding.UTF8.GetByteCount(key) <= ChromaCloudQuotas.MaxMetadataKeyBytes;
+
     /// <summary>
     /// Get the Chroma distance function, called space, for the given <paramref name="vectorProperty"/>.
     /// If none is configured, the default is cosine.

MEVD/src/Chroma/ChromaModelBuilder.cs

@@ -52,6 +52,13 @@ protected override void ValidateProperty(PropertyModel propertyModel, VectorStor
                 $"Property '{propertyModel.ModelName}' has the storage name '{propertyModel.StorageName}', but Chroma does not accept a metadata key that is empty or starts with # or $.");
         }
 
+        // Chroma drops the metadata keys that start with chroma: from every read: the property would always read back as null.
+        if (propertyModel is DataPropertyModel { StorageName: var storageName } && storageName.StartsWith("chroma:", StringComparison.Ordinal))
+        {
+            throw new NotSupportedException(
+                $"Property '{propertyModel.ModelName}' has the storage name '{storageName}', but Chroma drops the metadata keys that start with chroma: from every read.");
+        }
+
         switch (propertyModel)
         {
             case VectorPropertyModel vectorProperty:

MEVD/src/Chroma/README.md

@@ -26,10 +26,15 @@ For more information, see the [Microsoft.Extensions.VectorData documentation](ht
 
 - On Chroma Cloud, a vector search returns at most 300 results, `Skip` included, the default quota of Chroma Cloud: beyond that, Chroma Cloud answers with a quota error.
 - Chroma does not store empty lists, so an empty array or list comes back as `null`.
+- A data property whose storage name is empty, or starts with `#`, `$` or `chroma:`, throws a `NotSupportedException` when the collection object is created. Chroma rejects the first ones in a write, and drops the `chroma:` keys from every read, so such a property would always come back as `null`.
 - Array and list properties, like the namespaces of the `TextSearchStore` of Semantic Kernel, need Chroma 1.5.0 or later.
+- A collection created by another client without a space has the space l2. A search on it with a distance function of another space throws, except on Chroma 1.0.0 to 1.0.5: there the provider cannot see the space, and the search returns wrong scores without an error. Use Chroma 1.0.6 or later.
 - On self-hosted Chroma 1.0.21 to 1.5.9, a vector search can miss a record after an upsert, while a get still finds it. It is KD-49 in the [compatibility notes](https://github.com/ChromaDotNet/ChromaDB.Client/blob/main/docs/COMPATIBILITY.md#records-lost-after-an-update-of-their-embeddings) of ChromaDotNet.Client. The environment variable `RAYON_NUM_THREADS=1` on the server makes it much rarer. In the client, `ChromaUpsertStrategy.SkipUnchangedEmbeddings` writes a record whose vector does not change without its vector, so that write does not touch the vector index.
+- On self-hosted Chroma 1.5.9, while upserts of the same record run at the same time, a read can return a mix of two of them: the keys that one sets to `null` and the other writes. Once the writes end, the record is one of them.
+- On Chroma 1.5.9, with the cosine space and vectors that are nearly parallel, a vector search over all the records can miss some of them, and pages can repeat records. It comes from the approximate index of Chroma; with the l2 space the same records are all found.
 - On Chroma Cloud, a record has at most 32 metadata keys, and a null value counts as one. The provider writes every data property, null or not. A write with null values that would take a record beyond 32 keys throws an `ArgumentException`. With `WithReadsBeyondKeyQuota()`, the client reads such a record first and sends only the null values of the keys it has. Two writes of the same record at the same time can then mix.
 - On Chroma Cloud, a metadata key has at most 36 bytes in UTF-8. With a data property whose storage name is longer, the collection is created, but every upsert throws a `VectorStoreException` with the quota error of Chroma Cloud.
+- With two or more full-text properties, each has a BM25 index of its own, whose key is its storage name with `_bm25`. On Chroma Cloud, a storage name of more than 31 bytes makes that key longer than 36 bytes: the collection is created without that index, and a hybrid search on that property throws an `InvalidOperationException` that says why.
 - On Chroma Cloud, `Contains` on the full-text property throws an `ArgumentException` for a text of fewer than 3 characters. Once Chroma Cloud has indexed the records, such a filter would find no record, and its negation every record.
 
 The connection string does not set `WithUpsertStrategy` and `WithReadsBeyondKeyQuota()`. Set them on the options of a `ChromaClient` created in code, and pass it to the vector store:

MEVD/test/Chroma.UnitTests/ChromaCollectionTests.cs

@@ -120,6 +120,67 @@ public void RejectsAFullTextPropertyWhoseStorageNameStartsWithAHash()
         Assert.Throws<NotSupportedException>(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition }));
     }
 
+    [Theory]
+    [InlineData("chroma:tag")]
+    [InlineData("chroma:document")]
+    [InlineData("chroma:uri")]
+    [InlineData("chroma:")]
+    public void RejectsADataPropertyWhoseStorageNameStartsWithTheChromaPrefix(string storageName)
+    {
+        // Chroma drops the metadata keys that start with chroma: from every read: the property would always read back as null.
+        var definition = new VectorStoreCollectionDefinition
+        {
+            Properties =
+            [
+                new VectorStoreKeyProperty("Key", typeof(string)),
+                new VectorStoreDataProperty("Tag", typeof(string)) { StorageName = storageName },
+                new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), 4),
+            ]
+        };
+
+        var exception = Assert.Throws<NotSupportedException>(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition }));
+        Assert.Contains($"'{storageName}'", exception.Message);
+    }
+
+    [Fact]
+    public void RejectsAFullTextPropertyWhoseStorageNameStartsWithTheChromaPrefix()
+    {
+        // The full-text property stored as the document is also a metadata key: the client copies the document into it.
+        var definition = new VectorStoreCollectionDefinition
+        {
+            Properties =
+            [
+                new VectorStoreKeyProperty("Key", typeof(string)),
+                new VectorStoreDataProperty("Text", typeof(string)) { StorageName = "chroma:document", IsFullTextIndexed = true },
+                new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), 4),
+            ]
+        };
+
+        Assert.Throws<NotSupportedException>(() => new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition }));
+    }
+
+    [Theory]
+    [InlineData("Chroma:tag")]
+    [InlineData("CHROMA:tag")]
+    [InlineData("chroma_tag")]
+    [InlineData("tag:chroma:")]
+    public void AcceptsAStorageNameThatDoesNotStartWithTheChromaPrefix(string storageName)
+    {
+        // Chroma drops only the keys that start with chroma: in lower case.
+        var definition = new VectorStoreCollectionDefinition
+        {
+            Properties =
+            [
+                new VectorStoreKeyProperty("Key", typeof(string)),
+                new VectorStoreDataProperty("Tag", typeof(string)) { StorageName = storageName },
+                new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory<float>), 4),
+            ]
+        };
+
+        using var sut = new ChromaDynamicCollection(this._chromaClientMock.Object, TestCollectionName, ownsClient: false, new() { Definition = definition });
+        Assert.Equal(TestCollectionName, sut.Name);
+    }
+
     [Theory]
     [InlineData("tag#1")]
     [InlineData("price$usd")]
@@ -218,6 +279,21 @@ public void GetServiceOffersHybridSearchOnlyWithFullTextPropertiesOnChromaCloud(
         Assert.Null(noFullText.GetService(typeof(IKeywordHybridSearchable<ChromaHotel<string>>)));
     }
 
+    [Fact]
+    public void GetServiceOffersHybridSearchOnlyWithABm25IndexKeyThatChromaCloudTakes()
+    {
+        // Arrange: the client leaves out on Chroma Cloud a BM25 index whose key is beyond 36 bytes.
+        this._chromaClientMock
+            .Setup(x => x.Options)
+            .Returns(new ChromaConfigurationOptions("https://api.trychroma.com"));
+        using var oneFits = this.CreateCollection<string, LongFullTextHotel>();
+        using var noneFits = this.CreateCollection<string, TooLongFullTextHotel>();
+
+        // Act and assert.
+        Assert.NotNull(oneFits.GetService(typeof(IKeywordHybridSearchable<LongFullTextHotel>)));
+        Assert.Null(noneFits.GetService(typeof(IKeywordHybridSearchable<TooLongFullTextHotel>)));
+    }
+
     #endregion
 
     #region Get by key
@@ -400,7 +476,22 @@ public async Task HybridSearchThrowsWithoutABm25IndexAsync()
 
         // Act and assert.
         var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () => await sut.HybridSearchAsync(new ReadOnlyMemory<float>([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync());
-        Assert.Contains("Chroma Cloud", exception.Message);
+        Assert.Contains("Only Chroma Cloud has BM25 indexes.", exception.Message);
+    }
+
+    [Fact]
+    public async Task HybridSearchOnChromaCloudSaysTheCollectionWasCreatedWithoutTheBm25IndexAsync()
+    {
+        // Arrange: Chroma Cloud, with a key that fits but no index, as in a collection that another client created.
+        this._chromaClientMock
+            .Setup(x => x.Options)
+            .Returns(new ChromaConfigurationOptions("https://api.trychroma.com"));
+        using var sut = this.CreateHybridCollection<FullTextHotel>();
+
+        // Act and assert.
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () => await sut.HybridSearchAsync(new ReadOnlyMemory<float>([1, 2, 3, 4]), ["pool"], top: 1, cancellationToken: this._testCancellationToken).ToListAsync());
+        Assert.Contains("created without it", exception.Message);
+        Assert.DoesNotContain("Only Chroma Cloud", exception.Message);
     }
 
     [Fact]
@@ -414,6 +505,21 @@ public async Task HybridSearchThrowsWithoutAnIndexOnTheChosenPropertyAsync()
         Assert.Contains("'Review'", exception.Message);
     }
 
+    [Fact]
+    public async Task HybridSearchSaysWhyChromaCloudHasNoBm25IndexOnALongStorageNameAsync()
+    {
+        // Arrange: Chroma Cloud, with only the index of the property that fits.
+        this._chromaClientMock
+            .Setup(x => x.Options)
+            .Returns(new ChromaConfigurationOptions("https://api.trychroma.com"));
+        using var sut = this.CreateHybridCollection<LongFullTextHotel>(Bm25Index("Review_with_a_long_name_xxxxxxx_bm25", "Review_with_a_long_name_xxxxxxx"));
+
+        // Act and assert.
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () => await sut.HybridSearchAsync(new ReadOnlyMemory<float>([1, 2, 3, 4]), ["pool"], top: 1, new() { AdditionalProperty = h => h.Description }, this._testCancellationToken).ToListAsync());
+        Assert.Contains("'Description_with_a_long_name_xxx_bm25'", exception.Message);
+        Assert.Contains("36 bytes", exception.Message);
+    }
+
     [Fact]
     public async Task HybridSearchCannotReadVectorsWithEmbeddingGenerationAsync()
     {

MEVD/test/Chroma.UnitTests/ChromaHotel.cs

@@ -102,6 +102,39 @@ public sealed class TwoFullTextHotel
     public ReadOnlyMemory<float>? Embedding { get; set; }
 }
 
+// Two full-text properties, each with a BM25 index of its own: the key of the first, Description_with_a_long_name_xxx_bm25,
+// has 37 bytes, beyond the 36 of Chroma Cloud; the key of the second, Review_with_a_long_name_xxxxxxx_bm25, has 36.
+public sealed class LongFullTextHotel
+{
+    [VectorStoreKey]
+    public string HotelId { get; set; } = default!;
+
+    [VectorStoreData(IsFullTextIndexed = true, StorageName = "Description_with_a_long_name_xxx")]
+    public string? Description { get; set; }
+
+    [VectorStoreData(IsFullTextIndexed = true, StorageName = "Review_with_a_long_name_xxxxxxx")]
+    public string? Review { get; set; }
+
+    [VectorStoreVector(4)]
+    public ReadOnlyMemory<float>? Embedding { get; set; }
+}
+
+// As LongFullTextHotel, with both keys beyond the 36 bytes of Chroma Cloud.
+public sealed class TooLongFullTextHotel
+{
+    [VectorStoreKey]
+    public string HotelId { get; set; } = default!;
+
+    [VectorStoreData(IsFullTextIndexed = true, StorageName = "Description_with_a_long_name_xxx")]
+    public string? Description { get; set; }
+
+    [VectorStoreData(IsFullTextIndexed = true, StorageName = "Review_with_a_long_name_xxxxxxxx")]
+    public string? Review { get; set; }
+
+    [VectorStoreVector(4)]
+    public ReadOnlyMemory<float>? Embedding { get; set; }
+}
+
 public sealed class FullTextHotel
 {
     [VectorStoreKey]