From 78e5b1f6a8d349f30703b39fe7f027eb646b567c Mon Sep 17 00:00:00 2001 From: Luca Schiavone <16616566+Overvolted@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:27:19 +0200 Subject: [PATCH] Add Milvus vector data provider Add the Milvus.Client-backed package, typed and dynamic collections, scalar and array filtering, vector and BM25 hybrid search, dependency injection, documentation, and unit and conformance tests. Co-authored-by: Federico Laurianti --- CommunityToolkit.AI.slnx | 3 + Directory.Packages.props | 2 + MEVD/MEVD.slnf | 3 + MEVD/src/Milvus/AssemblyInfo.cs | 2 + MEVD/src/Milvus/Milvus.csproj | 31 + MEVD/src/Milvus/MilvusCollection.cs | 597 ++++++++++++++++++ .../Milvus/MilvusCollectionCreateMapping.cs | 93 +++ MEVD/src/Milvus/MilvusCollectionOptions.cs | 25 + .../Milvus/MilvusCollectionSearchMapping.cs | 17 + MEVD/src/Milvus/MilvusConstants.cs | 9 + MEVD/src/Milvus/MilvusDynamicCollection.cs | 38 ++ MEVD/src/Milvus/MilvusErrorHandler.cs | 23 + MEVD/src/Milvus/MilvusFieldMapping.cs | 129 ++++ MEVD/src/Milvus/MilvusFilterTranslator.cs | 201 ++++++ MEVD/src/Milvus/MilvusMapper.cs | 89 +++ MEVD/src/Milvus/MilvusModelBuilder.cs | 117 ++++ .../MilvusServiceCollectionExtensions.cs | 328 ++++++++++ MEVD/src/Milvus/MilvusVectorStore.cs | 151 +++++ MEVD/src/Milvus/MilvusVectorStoreOptions.cs | 32 + MEVD/src/Milvus/README.md | 63 ++ MEVD/src/Milvus/SharedMilvusClient.cs | 67 ++ .../Milvus.ConformanceTests.csproj | 25 + .../MilvusArrayFilterTests.cs | 72 +++ .../MilvusCollectionManagementTests.cs | 14 + .../MilvusDatabaseTests.cs | 45 ++ .../MilvusDependencyInjectionTests.cs | 99 +++ .../MilvusDistanceFunctionTests.cs | 19 + .../MilvusEmbeddingGenerationTests.cs | 62 ++ .../MilvusEnsureCollectionTests.cs | 87 +++ .../MilvusFilterTests.cs | 15 + .../MilvusFullTextTests.cs | 205 ++++++ .../MilvusHybridSearchTests.cs | 99 +++ .../MilvusIndexKindTests.cs | 15 + .../MilvusNullComparisonTests.cs | 110 ++++ .../MilvusRoundTripTests.cs | 119 ++++ .../MilvusTestSuiteImplementationTests.cs | 8 + .../ModelTests/MilvusBasicModelTests.cs | 36 ++ .../ModelTests/MilvusDynamicModelTests.cs | 36 ++ .../ModelTests/MilvusMultiVectorModelTests.cs | 18 + .../ModelTests/MilvusNoDataModelTests.cs | 19 + .../ModelTests/MilvusNoVectorModelTests.cs | 29 + .../Support/MilvusFixture.cs | 11 + .../Support/MilvusTestStore.cs | 52 ++ .../TypeTests/MilvusDataTypeTests.cs | 24 + .../TypeTests/MilvusEmbeddingTypeTests.cs | 20 + .../TypeTests/MilvusKeyTypeTests.cs | 20 + MEVD/test/Milvus.UnitTests/.editorconfig | 6 + .../Milvus.UnitTests/Milvus.UnitTests.csproj | 36 ++ .../MilvusArrayFilterTests.cs | 66 ++ .../test/Milvus.UnitTests/MilvusArrayTests.cs | 86 +++ .../MilvusClientLifetimeTests.cs | 161 +++++ .../Milvus.UnitTests/MilvusCollectionTests.cs | 72 +++ .../Milvus.UnitTests/MilvusDatabaseTests.cs | 74 +++ .../MilvusEnsureCollectionTests.cs | 128 ++++ .../MilvusFilterTranslatorTests.cs | 146 +++++ .../Milvus.UnitTests/MilvusFullTextTests.cs | 192 ++++++ .../MilvusHybridSearchTests.cs | 115 ++++ .../Milvus.UnitTests/MilvusMappingTests.cs | 170 +++++ .../MilvusStringEscapingTests.cs | 15 + .../test/Milvus.UnitTests/MilvusTestRecord.cs | 25 + README.md | 4 + 61 files changed, 4575 insertions(+) create mode 100644 MEVD/src/Milvus/AssemblyInfo.cs create mode 100644 MEVD/src/Milvus/Milvus.csproj create mode 100644 MEVD/src/Milvus/MilvusCollection.cs create mode 100644 MEVD/src/Milvus/MilvusCollectionCreateMapping.cs create mode 100644 MEVD/src/Milvus/MilvusCollectionOptions.cs create mode 100644 MEVD/src/Milvus/MilvusCollectionSearchMapping.cs create mode 100644 MEVD/src/Milvus/MilvusConstants.cs create mode 100644 MEVD/src/Milvus/MilvusDynamicCollection.cs create mode 100644 MEVD/src/Milvus/MilvusErrorHandler.cs create mode 100644 MEVD/src/Milvus/MilvusFieldMapping.cs create mode 100644 MEVD/src/Milvus/MilvusFilterTranslator.cs create mode 100644 MEVD/src/Milvus/MilvusMapper.cs create mode 100644 MEVD/src/Milvus/MilvusModelBuilder.cs create mode 100644 MEVD/src/Milvus/MilvusServiceCollectionExtensions.cs create mode 100644 MEVD/src/Milvus/MilvusVectorStore.cs create mode 100644 MEVD/src/Milvus/MilvusVectorStoreOptions.cs create mode 100644 MEVD/src/Milvus/README.md create mode 100644 MEVD/src/Milvus/SharedMilvusClient.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/Milvus.ConformanceTests.csproj create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusArrayFilterTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusCollectionManagementTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusDatabaseTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusDependencyInjectionTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusDistanceFunctionTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusEmbeddingGenerationTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusEnsureCollectionTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusFilterTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusFullTextTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusHybridSearchTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusIndexKindTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusNullComparisonTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusRoundTripTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/MilvusTestSuiteImplementationTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusBasicModelTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusDynamicModelTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusMultiVectorModelTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoDataModelTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoVectorModelTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/Support/MilvusFixture.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/Support/MilvusTestStore.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusDataTypeTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusEmbeddingTypeTests.cs create mode 100644 MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusKeyTypeTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/.editorconfig create mode 100644 MEVD/test/Milvus.UnitTests/Milvus.UnitTests.csproj create mode 100644 MEVD/test/Milvus.UnitTests/MilvusArrayFilterTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusArrayTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusClientLifetimeTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusCollectionTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusDatabaseTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusEnsureCollectionTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusFilterTranslatorTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusFullTextTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusHybridSearchTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusMappingTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusStringEscapingTests.cs create mode 100644 MEVD/test/Milvus.UnitTests/MilvusTestRecord.cs diff --git a/CommunityToolkit.AI.slnx b/CommunityToolkit.AI.slnx index 35e2b7c..a459dd0 100644 --- a/CommunityToolkit.AI.slnx +++ b/CommunityToolkit.AI.slnx @@ -13,6 +13,7 @@ + @@ -32,6 +33,8 @@ + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 4f15290..5684c14 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,6 +17,7 @@ + @@ -46,6 +47,7 @@ + diff --git a/MEVD/MEVD.slnf b/MEVD/MEVD.slnf index 80d205a..ef1de46 100644 --- a/MEVD/MEVD.slnf +++ b/MEVD/MEVD.slnf @@ -8,6 +8,7 @@ "MEVD/src/AzureDocumentDB/AzureDocumentDB.csproj", "MEVD/src/Chroma/Chroma.csproj", "MEVD/src/InMemory/InMemory.csproj", + "MEVD/src/Milvus/Milvus.csproj", "MEVD/src/PgVector/PgVector.csproj", "MEVD/src/Qdrant/Qdrant.csproj", "MEVD/src/Redis/Redis.csproj", @@ -25,6 +26,8 @@ "MEVD/test/Chroma.ConformanceTests/Chroma.ConformanceTests.csproj", "MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj", "MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj", + "MEVD/test/Milvus.UnitTests/Milvus.UnitTests.csproj", + "MEVD/test/Milvus.ConformanceTests/Milvus.ConformanceTests.csproj", "MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj", "MEVD/test/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj", "MEVD/test/Qdrant.UnitTests/Qdrant.UnitTests.csproj", diff --git a/MEVD/src/Milvus/AssemblyInfo.cs b/MEVD/src/Milvus/AssemblyInfo.cs new file mode 100644 index 0000000..53616c2 --- /dev/null +++ b/MEVD/src/Milvus/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. diff --git a/MEVD/src/Milvus/Milvus.csproj b/MEVD/src/Milvus/Milvus.csproj new file mode 100644 index 0000000..c786b80 --- /dev/null +++ b/MEVD/src/Milvus/Milvus.csproj @@ -0,0 +1,31 @@ + + + + 1.0.0-preview + CommunityToolkit.VectorData.Milvus + $(AssemblyName) + net10.0;net8.0;netstandard2.0;net462 + + true + + Milvus provider for Microsoft.Extensions.VectorData + Milvus provider for Microsoft.Extensions.VectorData by the .NET Community Toolkit + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/MEVD/src/Milvus/MilvusCollection.cs b/MEVD/src/Milvus/MilvusCollection.cs new file mode 100644 index 0000000..e2c3bd4 --- /dev/null +++ b/MEVD/src/Milvus/MilvusCollection.cs @@ -0,0 +1,597 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using Milvus.Client; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// Service for storing and retrieving vector records, that uses Milvus as the underlying storage. +/// +/// The data type of the record key. Can be , or . +/// The data model to use for adding, updating and retrieving data from storage. +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class MilvusCollection : VectorStoreCollection, IKeywordHybridSearchable + where TKey : notnull + where TRecord : class +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// Metadata about vector store record collection. + private readonly VectorStoreCollectionMetadata _collectionMetadata; + + /// The default options for vector search. + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + + private static readonly HybridSearchOptions s_defaultHybridSearchOptions = new(); + + // The two searches of a hybrid search rank at least this many records each. When they rank only the first top + skip, + // a record found by both can miss one of the rankings, as when records tie in the vector search, and the fusion ranks it lower. + private const int HybridSearchCandidates = 10; + + // The most records a Milvus search ranks. + private const int MaxSearchLimit = 16384; + + // How long EnsureCollectionExistsAsync waits for the collection to load: a little longer than the load timeout of the + // server, 600 seconds in the default configuration of Milvus 2.6.25 (queryCoord.loadTimeoutSeconds), so that the + // provider does not give up on a load sooner than the server; a load that never ends still stops. + internal TimeSpan LoadTimeout { get; set; } = TimeSpan.FromMinutes(11); + + /// The name of the upsert operation for telemetry purposes. + private const string UpsertName = "Upsert"; + + /// The name of the Delete operation for telemetry purposes. + private const string DeleteName = "Delete"; + + /// Milvus client that can be used to manage the collections and records in a Milvus store. + private readonly SharedMilvusClient _milvusClient; + + /// Milvus client of the records of the collection. + private readonly global::Milvus.Client.MilvusCollection _records; + + /// The model for this collection. + private readonly CollectionModel _model; + + /// A mapper to use for converting between Milvus records and consumer models. + private readonly MilvusMapper _mapper; + + /// Whether the collection was disposed: it releases its share of the client only once. + private int _disposed; + + /// + /// Initializes a new instance of the class. + /// + /// Milvus client that can be used to manage the collections and records in a Milvus store. + /// The name of the collection that this will access. + /// A value indicating whether is disposed when the collection is disposed. + /// Optional configuration options for this class. + /// Thrown if the is null. + /// Thrown for any misconfigured options. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate MilvusDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate MilvusDynamicCollection instead")] + public MilvusCollection(MilvusClient milvusClient, string name, bool ownsClient, MilvusCollectionOptions? options = null) + : this(() => new SharedMilvusClient(milvusClient, ownsClient), name, options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Milvus client factory. + /// The name of the collection that this will access. + /// Optional configuration options for this class. + /// Thrown if the is null. + /// Thrown for any misconfigured options. + [RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary, instantiate MilvusDynamicCollection instead.")] + [RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary, instantiate MilvusDynamicCollection instead")] + internal MilvusCollection(Func clientFactory, string name, MilvusCollectionOptions? options = null) + : this( + clientFactory, + name, + static options => typeof(TRecord) == typeof(Dictionary) + ? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(MilvusDynamicCollection))) + : new MilvusModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator), + options) + { + } + + internal MilvusCollection(Func clientFactory, string name, Func modelFactory, MilvusCollectionOptions? options) + { + // Verify. + Throw.IfNull(clientFactory); + + // The collection holds the client from here: when the rest of the construction throws, it releases the client, + // so that a client it was to own is disposed, and a client it shares with its vector store is given back. + _milvusClient = clientFactory(); + try + { + Throw.IfNullOrWhitespace(name); + + if (typeof(TKey) != typeof(long) && typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object)) + { + throw new NotSupportedException("Only long, string and Guid keys are supported."); + } + + options ??= MilvusCollectionOptions.Default; + + // Assign. + Name = name; + _model = modelFactory(options); + _mapper = new MilvusMapper(_model); + } + catch + { + _milvusClient.Dispose(); + throw; + } + + _records = _milvusClient.Client.GetCollection(name); + + _collectionMetadata = new() + { + VectorStoreSystemName = MilvusConstants.VectorStoreSystemName, + VectorStoreName = _milvusClient.DatabaseName, + CollectionName = name + }; + } + + /// + protected override void Dispose(bool disposing) + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _milvusClient.Dispose(); + } + + base.Dispose(disposing); + } + + /// + public override string Name { get; } + + /// + public override Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("CollectionExists", () => _milvusClient.Client.HasCollectionAsync(Name, cancellationToken: cancellationToken)); + + /// + public override Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("EnsureCollectionExists", async () => + { + // The indexes are created with the collection. Those of an existing collection stay as they are: Milvus rejects + // an index with other settings on an indexed field, or on a field the collection does not have. + if (!await _milvusClient.Client.HasCollectionAsync(Name, cancellationToken: cancellationToken).ConfigureAwait(false)) + { + await _milvusClient.Client.CreateCollectionAsync(Name, MilvusCollectionCreateMapping.MapCollectionSchema(_model), + cancellationToken: cancellationToken).ConfigureAwait(false); + foreach (var vector in _model.VectorProperties) + { + var indexType = MilvusCollectionCreateMapping.GetIndexType(vector); + var parameters = indexType == IndexType.Hnsw + ? new Dictionary { ["M"] = "16", ["efConstruction"] = "200" } : null; + await _records.CreateIndexAsync(vector.StorageName, indexType, + MilvusCollectionCreateMapping.GetMetric(vector), extraParams: parameters, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + foreach (var property in MilvusCollectionCreateMapping.GetFullTextProperties(_model)) + { + await _records.CreateIndexAsync(MilvusCollectionCreateMapping.GetBm25FieldName(property), IndexType.SparseInvertedIndex, + SimilarityMetricType.Bm25, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + await _records.LoadAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + await _records.WaitForCollectionLoadAsync(timeout: LoadTimeout, cancellationToken: cancellationToken).ConfigureAwait(false); + }); + + /// + public override Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default) + => RunOperationAsync("DeleteCollection", async () => + { + if (await _milvusClient.Client.HasCollectionAsync(Name, cancellationToken: cancellationToken).ConfigureAwait(false)) + { + await _records.DropAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + }); + + /// + public override async Task GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + var records = await GetAsync([key], options, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); + return records.FirstOrDefault(); + } + + /// + public override async IAsyncEnumerable GetAsync( + IEnumerable keys, + RecordRetrievalOptions? options = default, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "Get"; + + Throw.IfNull(keys); + + var includeVectors = options?.IncludeVectors ?? false; + if (includeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var keyList = keys.ToList(); + if (keyList.Count == 0) + { + yield break; + } + var parameters = CreateQueryParameters(includeVectors); + var entries = await RunOperationAsync(OperationName, + () => _records.QueryRowsAsync(KeyFilter(keyList), parameters, cancellationToken)).ConfigureAwait(false); + foreach (var entry in entries) + { + yield return _mapper.MapFromStorageToDataModel(entry, includeVectors); + } + } + + /// + public override Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + Throw.IfNull(key); + + return DeleteAsync([key], cancellationToken); + } + + /// + public override Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + Throw.IfNull(keys); + + var keyList = keys.ToList(); + return keyList.Count == 0 ? Task.CompletedTask + : RunOperationAsync(DeleteName, () => _records.DeleteAsync(KeyFilter(keyList), cancellationToken: cancellationToken)); + } + + /// + public override async Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + Throw.IfNull(record); + + await UpsertAsync([record], cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + Throw.IfNull(records); + + IReadOnlyList? recordsList = null; + + // If an embedding generator is defined, invoke it once per property for all records. + GeneratedEmbeddings>?[]? generatedEmbeddings = null; + + var vectorPropertyCount = _model.VectorProperties.Count; + for (var i = 0; i < vectorPropertyCount; i++) + { + var vectorProperty = _model.VectorProperties[i]; + + if (MilvusModelBuilder.IsVectorPropertyTypeValidCore(vectorProperty.Type, out _)) + { + continue; + } + + // We have a vector property whose type isn't natively supported - we need to generate embeddings. + Debug.Assert(vectorProperty.EmbeddingGenerator is not null); + + if (recordsList is null) + { + recordsList = records is IReadOnlyList r ? r : records.ToList(); + + if (recordsList.Count == 0) + { + return; + } + + records = recordsList; + } + + // TODO: Ideally we'd group together vector properties using the same generator (and with the same input and output properties), + // and generate embeddings for them in a single batch. That's some more complexity though. + generatedEmbeddings ??= new GeneratedEmbeddings>?[vectorPropertyCount]; + generatedEmbeddings[i] = (GeneratedEmbeddings>)await vectorProperty.GenerateEmbeddingsAsync(records.Select(r => vectorProperty.GetValueAsObject(r)), cancellationToken).ConfigureAwait(false); + } + + var keyProperty = _model.KeyProperty; + var rows = new List>(); + var index = 0; + foreach (var record in records) + { + Throw.IfNull(record); + if (keyProperty.IsAutoGenerated && keyProperty.GetValue(record) == Guid.Empty) + { + keyProperty.SetValue(record, Guid.NewGuid()); + } + rows.Add(_mapper.MapFromDataToStorageModel(record, index++, generatedEmbeddings)); + } + if (rows.Count > 0) + { + await RunOperationAsync(UpsertName, () => _records.UpsertAsync(MilvusFieldMapping.CreateColumns(_model, rows), cancellationToken: cancellationToken)).ConfigureAwait(false); + } + } + + #region Search + + /// + public override async IAsyncEnumerable> SearchAsync( + TInput searchValue, + int top, + VectorSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(searchValue); + Throw.IfLessThan(top, 1); + + options ??= s_defaultVectorSearchOptions; + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(options); + + var vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + if (vector.Length != vectorProperty.Dimensions) + { + throw new ArgumentException($"Search vector must have {vectorProperty.Dimensions} dimensions.", nameof(searchValue)); + } + var parameters = new SearchParameters + { + Offset = options.Skip, + Expression = options.Filter is null ? null : new MilvusFilterTranslator().Translate(options.Filter, _model), + ConsistencyLevel = ConsistencyLevel.Strong, + }; + foreach (var field in GetOutputFields(options.IncludeVectors)) + { + parameters.OutputFields.Add(field); + } + if (MilvusCollectionCreateMapping.GetIndexType(vectorProperty) == IndexType.Hnsw) + { + parameters.ExtraParameters["ef"] = Math.Max(64, top + options.Skip).ToString(System.Globalization.CultureInfo.InvariantCulture); + } + var results = await RunOperationAsync("Search", () => SearchAllowingEmptyResultsAsync( + () => _records.SearchAsync(vectorProperty.StorageName, [vector], + MilvusCollectionCreateMapping.GetMetric(vectorProperty), top, parameters, cancellationToken), + parameters.OutputFields)).ConfigureAwait(false); + if (IsEmpty(results)) + { + yield break; + } + foreach (var hit in results.GetHits()) + { + var score = MilvusCollectionSearchMapping.MapScore(hit.Score, vectorProperty.DistanceFunction); + if (MilvusCollectionSearchMapping.PassesThreshold(score, options.ScoreThreshold, vectorProperty.DistanceFunction)) + { + yield return new VectorSearchResult(MapHit(hit, options.IncludeVectors), score); + } + } + } + + /// + public async IAsyncEnumerable> HybridSearchAsync( + TInput searchValue, + ICollection keywords, + int top, + HybridSearchOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + where TInput : notnull + { + Throw.IfNull(searchValue); + Throw.IfNull(keywords); + Throw.IfLessThan(top, 1); + + options ??= s_defaultHybridSearchOptions; + if ((long)top + options.Skip > MaxSearchLimit) + { + throw new ArgumentOutOfRangeException(nameof(top), + $"Milvus hybrid search ranks at most {MaxSearchLimit} records, the skipped ones included: top + Skip is {(long)top + options.Skip}."); + } + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + var vectorProperty = _model.GetVectorPropertyOrSingle(new() { VectorProperty = options.VectorProperty }); + var textProperty = _model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty); + var filter = options.Filter is null ? null : new MilvusFilterTranslator().Translate(options.Filter, _model); + + var vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false); + if (vector.Length != vectorProperty.Dimensions) + { + throw new ArgumentException($"Search vector must have {vectorProperty.Dimensions} dimensions.", nameof(searchValue)); + } + + // A vector search and a BM25 search of the keywords, both filtered, fused with Reciprocal Rank Fusion. + // Milvus has no offset for hybrid search: the skipped records are asked for too, and dropped. + var limit = top + options.Skip; + var candidates = Math.Max(limit, HybridSearchCandidates); + var vectorSearch = new VectorAnnSearchRequest(vectorProperty.StorageName, [vector], + MilvusCollectionCreateMapping.GetMetric(vectorProperty), candidates) + { + Expression = filter, + }; + if (MilvusCollectionCreateMapping.GetIndexType(vectorProperty) == IndexType.Hnsw) + { + vectorSearch.ExtraParameters["ef"] = Math.Max(64, candidates).ToString(System.Globalization.CultureInfo.InvariantCulture); + } + var keywordSearch = new TextAnnSearchRequest(MilvusCollectionCreateMapping.GetBm25FieldName(textProperty), + [string.Join(" ", keywords)], candidates) + { + Expression = filter, + }; + var parameters = new HybridSearchParameters { ConsistencyLevel = ConsistencyLevel.Strong }; + foreach (var field in GetOutputFields(options.IncludeVectors)) + { + parameters.OutputFields.Add(field); + } + var results = await RunOperationAsync("HybridSearch", () => SearchAllowingEmptyResultsAsync( + () => _records.HybridSearchAsync([vectorSearch, keywordSearch], new RrfReranker(60), limit, parameters, cancellationToken), + parameters.OutputFields)).ConfigureAwait(false); + if (IsEmpty(results)) + { + yield break; + } + foreach (var hit in results.GetHits().Skip(options.Skip)) + { + // A higher RRF score ranks higher: the threshold is a lower bound. + if (options.ScoreThreshold is not { } threshold || hit.Score >= threshold) + { + yield return new VectorSearchResult(MapHit(hit, options.IncludeVectors), hit.Score); + } + } + } + + // Milvus.Client 2.6.0 cannot decode untyped output fields in an empty response. + // Retry without output fields to verify that this really is an empty search; + // do not hide the SDK error if the retry finds any records. + private static async Task SearchAllowingEmptyResultsAsync(Func> search, IList outputFields) + { + try + { + return await search().ConfigureAwait(false); + } + catch (NotSupportedException exception) when (exception.Message == "Cannot convert None FieldData to Field") + { + outputFields.Clear(); + var retry = await search().ConfigureAwait(false); + if (!IsEmpty(retry)) + { + throw; + } + return retry; + } + } + + private static bool IsEmpty(SearchResults results) => (results.Ids.LongIds?.Count ?? results.Ids.StringIds?.Count ?? 0) == 0; + + private TRecord MapHit(SearchHit hit, bool includeVectors) + { + var row = hit.Fields.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + row[_model.KeyProperty.StorageName] = hit.Id; + return _mapper.MapFromStorageToDataModel(row, includeVectors); + } + + private static async ValueTask> GetSearchVectorAsync(TInput searchValue, VectorPropertyModel vectorProperty, CancellationToken cancellationToken) + where TInput : notnull + => searchValue switch + { + float[] array => array, + ReadOnlyMemory r => r, + Embedding e => e.Vector, + _ when vectorProperty.EmbeddingGenerationDispatcher is not null + => ((Embedding)await vectorProperty.GenerateEmbeddingAsync(searchValue, cancellationToken).ConfigureAwait(false)).Vector, + + _ => vectorProperty.EmbeddingGenerator is null + ? throw new NotSupportedException(VectorDataStrings.InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(searchValue.GetType(), MilvusModelBuilder.SupportedVectorTypes)) + : throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType())) + }; + + #endregion Search + + /// + public override async IAsyncEnumerable GetAsync(Expression> filter, int top, + FilteredRecordRetrievalOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Throw.IfNull(filter); + Throw.IfLessThan(top, 1); + + options ??= new(); + + if (options.IncludeVectors && _model.EmbeddingGenerationRequired) + { + throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration); + } + + if (options.OrderBy?.Invoke(new()).Values is { Count: > 0 }) + { + throw new NotSupportedException("Milvus does not support ordering."); + } + + var milvusFilter = new MilvusFilterTranslator().Translate(filter, _model); + var parameters = CreateQueryParameters(options.IncludeVectors); + parameters.Limit = top; + parameters.Offset = options.Skip; + var entries = await RunOperationAsync("Get", + () => _records.QueryRowsAsync(milvusFilter, parameters, cancellationToken)).ConfigureAwait(false); + foreach (var entry in entries) + { + yield return _mapper.MapFromStorageToDataModel(entry, options.IncludeVectors); + } + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreCollectionMetadata) ? _collectionMetadata : + serviceType == typeof(MilvusClient) ? _milvusClient.Client : + // Hybrid search needs a full-text indexed property. + serviceType == typeof(IKeywordHybridSearchable) ? (MilvusCollectionCreateMapping.GetFullTextProperties(_model).Any() ? this : null) : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + private string KeyFilter(IEnumerable keys) + { + var literals = keys.Select(key => + { + Throw.IfNull(key); + if (key.GetType() != _model.KeyProperty.Type) + { + throw new ArgumentException($"Expected key type '{_model.KeyProperty.Type}'.", nameof(keys)); + } + return MilvusFieldMapping.Literal(key); + }); + return $"{_model.KeyProperty.StorageName} in [{string.Join(", ", literals)}]"; + } + + private IEnumerable GetOutputFields(bool includeVectors) + => new[] { _model.KeyProperty.StorageName }.Concat(_model.DataProperties.Select(p => p.StorageName)) + .Concat(includeVectors ? _model.VectorProperties.Select(p => p.StorageName) : []); + + private QueryParameters CreateQueryParameters(bool includeVectors) + { + var parameters = new QueryParameters { ConsistencyLevel = ConsistencyLevel.Strong }; + foreach (var field in GetOutputFields(includeVectors)) + { + parameters.OutputFields.Add(field); + } + return parameters; + } + + /// + /// Run the given operation and wrap any with . + /// + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + private Task RunOperationAsync(string operationName, Func operation) + => MilvusErrorHandler.RunAsync(_collectionMetadata, operationName, operation); + + /// + /// Run the given operation and wrap any with . + /// + /// The response type of the operation. + /// The type of database operation being run. + /// The operation to run. + /// The result of the operation. + // Awaited here rather than returned: otherwise the NativeAOT compiler of .NET 10 fails on the state machine of the handler (dotnet/runtime#120847). + private async Task RunOperationAsync(string operationName, Func> operation) + => await MilvusErrorHandler.RunAsync(_collectionMetadata, operationName, operation).ConfigureAwait(false); +} diff --git a/MEVD/src/Milvus/MilvusCollectionCreateMapping.cs b/MEVD/src/Milvus/MilvusCollectionCreateMapping.cs new file mode 100644 index 0000000..2c3ed98 --- /dev/null +++ b/MEVD/src/Milvus/MilvusCollectionCreateMapping.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.Client; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Milvus; + +internal static class MilvusCollectionCreateMapping +{ + public static IReadOnlyList MapCollectionDefinition(CollectionModel model) + { + var key = model.KeyProperty; + List fields = [key.Type == typeof(long) + ? FieldSchema.Create(key.StorageName, isPrimaryKey: true) + : FieldSchema.CreateVarchar(key.StorageName, 65535, isPrimaryKey: true)]; + foreach (var property in model.DataProperties) + { + var type = Nullable.GetUnderlyingType(property.Type) ?? property.Type; + var nullable = !property.Type.IsValueType || Nullable.GetUnderlyingType(property.Type) is not null; + if (MilvusFieldMapping.GetElementType(type) is { } element) + { + var name = property.StorageName; + var field = element == typeof(string) ? FieldSchema.CreateVarcharArray(name, 4096, 65535, nullable: true) + : element == typeof(bool) ? FieldSchema.CreateArray(name, 4096, nullable: true) + : element == typeof(int) ? FieldSchema.CreateArray(name, 4096, nullable: true) + : element == typeof(long) ? FieldSchema.CreateArray(name, 4096, nullable: true) + : element == typeof(float) ? FieldSchema.CreateArray(name, 4096, nullable: true) + : element == typeof(double) ? FieldSchema.CreateArray(name, 4096, nullable: true) + : throw new NotSupportedException($"Unsupported Milvus array element type '{element}'."); + fields.Add(field); + continue; + } + fields.Add(type == typeof(string) + ? FieldSchema.CreateVarchar(property.StorageName, 65535, nullable: true, enableAnalyzer: property.IsFullTextIndexed) + : FieldSchema.Create(property.StorageName, MilvusFieldMapping.GetDataType(type), nullable: nullable)); + } + foreach (var vector in model.VectorProperties) + { + fields.Add(FieldSchema.CreateFloatVector(vector.StorageName, vector.Dimensions)); + } + foreach (var property in GetFullTextProperties(model)) + { + fields.Add(FieldSchema.CreateSparseFloatVector(GetBm25FieldName(property))); + } + return fields; + } + + /// + /// Maps the model to the schema of the collection: its fields, and for each full-text indexed property, a BM25 + /// function that Milvus runs on the text to fill the hidden BM25 field. + /// + public static CollectionSchema MapCollectionSchema(CollectionModel model) + { + var schema = new CollectionSchema(); + foreach (var field in MapCollectionDefinition(model)) + { + schema.Fields.Add(field); + } + foreach (var property in GetFullTextProperties(model)) + { + var bm25FieldName = GetBm25FieldName(property); + schema.Functions.Add(FunctionSchema.CreateBm25(bm25FieldName, property.StorageName, bm25FieldName)); + } + return schema; + } + + public static IEnumerable GetFullTextProperties(CollectionModel model) + => model.DataProperties.Where(p => p.IsFullTextIndexed); + + /// + /// Gets the name of the hidden sparse field that holds the BM25 vectors of a full-text indexed property, and of + /// the BM25 function that fills it: the storage name of the property with _bm25. + /// + public static string GetBm25FieldName(DataPropertyModel property) => property.StorageName + "_bm25"; + + public static SimilarityMetricType GetMetric(VectorPropertyModel property) => property.DistanceFunction switch + { + null or DistanceFunction.CosineSimilarity => SimilarityMetricType.Cosine, + DistanceFunction.DotProductSimilarity => SimilarityMetricType.Ip, + DistanceFunction.EuclideanSquaredDistance or DistanceFunction.EuclideanDistance => SimilarityMetricType.L2, + _ => throw new NotSupportedException($"Milvus does not support distance function '{property.DistanceFunction}'."), + }; + + public static IndexType GetIndexType(VectorPropertyModel property) => property.IndexKind switch + { + null => IndexType.AutoIndex, + IndexKind.Flat => IndexType.Flat, + IndexKind.Hnsw => IndexType.Hnsw, + _ => throw new NotSupportedException($"Milvus does not support index kind '{property.IndexKind}'."), + }; +} diff --git a/MEVD/src/Milvus/MilvusCollectionOptions.cs b/MEVD/src/Milvus/MilvusCollectionOptions.cs new file mode 100644 index 0000000..1a05351 --- /dev/null +++ b/MEVD/src/Milvus/MilvusCollectionOptions.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// Options when creating a . +/// +public sealed class MilvusCollectionOptions : VectorStoreCollectionOptions +{ + internal static readonly MilvusCollectionOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public MilvusCollectionOptions() + { + } + + internal MilvusCollectionOptions(MilvusCollectionOptions? source) : base(source) + { + } +} diff --git a/MEVD/src/Milvus/MilvusCollectionSearchMapping.cs b/MEVD/src/Milvus/MilvusCollectionSearchMapping.cs new file mode 100644 index 0000000..12d98b8 --- /dev/null +++ b/MEVD/src/Milvus/MilvusCollectionSearchMapping.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Milvus; + +internal static class MilvusCollectionSearchMapping +{ + // For L2, Milvus returns the squared Euclidean distance: the Euclidean distance is its square root. + public static double MapScore(float score, string? distanceFunction) + => distanceFunction == DistanceFunction.EuclideanDistance ? Math.Sqrt(score) : score; + + public static bool PassesThreshold(double score, double? threshold, string? distanceFunction) + => threshold is null || (distanceFunction is DistanceFunction.EuclideanSquaredDistance or DistanceFunction.EuclideanDistance + ? score <= threshold.Value : score >= threshold.Value); +} diff --git a/MEVD/src/Milvus/MilvusConstants.cs b/MEVD/src/Milvus/MilvusConstants.cs new file mode 100644 index 0000000..877ea52 --- /dev/null +++ b/MEVD/src/Milvus/MilvusConstants.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace CommunityToolkit.VectorData.Milvus; + +internal static class MilvusConstants +{ + internal const string VectorStoreSystemName = "milvus"; +} diff --git a/MEVD/src/Milvus/MilvusDynamicCollection.cs b/MEVD/src/Milvus/MilvusDynamicCollection.cs new file mode 100644 index 0000000..70be65a --- /dev/null +++ b/MEVD/src/Milvus/MilvusDynamicCollection.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.Client; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// Represents a collection of vector store records in a Milvus database, mapped to a dynamic Dictionary<string, object?>. +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public sealed class MilvusDynamicCollection : MilvusCollection> +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + /// + /// Initializes a new instance of the class. + /// + /// Milvus client that can be used to manage the collections and points in a Milvus store. + /// The name of the collection. + /// A value indicating whether is disposed when the collection is disposed. + /// Optional configuration options for this class. + public MilvusDynamicCollection(MilvusClient milvusClient, string name, bool ownsClient, MilvusCollectionOptions options) + : this(() => new SharedMilvusClient(milvusClient, ownsClient), name, options) + { + } + + internal MilvusDynamicCollection(Func clientFactory, string name, MilvusCollectionOptions options) + : base( + clientFactory, + name, + static options => new MilvusModelBuilder() + .BuildDynamic( + options.Definition ?? throw new ArgumentException("Definition is required for dynamic collections"), + options.EmbeddingGenerator), + options) + { + } +} diff --git a/MEVD/src/Milvus/MilvusErrorHandler.cs b/MEVD/src/Milvus/MilvusErrorHandler.cs new file mode 100644 index 0000000..970e933 --- /dev/null +++ b/MEVD/src/Milvus/MilvusErrorHandler.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Grpc.Core; +using Microsoft.Extensions.VectorData; +using Milvus.Client; + +namespace CommunityToolkit.VectorData.Milvus; + +// Milvus.Client reports server errors as MilvusException, gRPC errors as RpcException, and a wait that runs out of time, +// such as the wait for a collection to load, as TimeoutException. +internal static class MilvusErrorHandler +{ + public static Task RunAsync(VectorStoreCollectionMetadata metadata, string operationName, Func operation) + => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, + () => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, + () => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, operation))); + + public static Task RunAsync(VectorStoreCollectionMetadata metadata, string operationName, Func> operation) + => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, + () => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, + () => VectorStoreErrorHandler.RunOperationAsync(metadata, operationName, operation))); +} diff --git a/MEVD/src/Milvus/MilvusFieldMapping.cs b/MEVD/src/Milvus/MilvusFieldMapping.cs new file mode 100644 index 0000000..257ca83 --- /dev/null +++ b/MEVD/src/Milvus/MilvusFieldMapping.cs @@ -0,0 +1,129 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Text; +using Milvus.Client; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Milvus; + +internal static class MilvusFieldMapping +{ + public static MilvusDataType GetDataType(Type type) + => type == typeof(string) ? MilvusDataType.VarChar + : type == typeof(bool) ? MilvusDataType.Bool + : type == typeof(int) ? MilvusDataType.Int32 + : type == typeof(long) ? MilvusDataType.Int64 + : type == typeof(float) ? MilvusDataType.Float + : type == typeof(double) ? MilvusDataType.Double + : throw new NotSupportedException($"Unsupported Milvus field type '{type}'."); + + internal static Type? GetElementType(Type type) + => type.IsArray ? type.GetElementType() + : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>) ? type.GenericTypeArguments[0] : null; + + internal static object? ReadValue(object? value, Type type) + { + if (value is null || GetElementType(type) is not { } element) + { + return value; + } + return element == typeof(string) ? ReadArray(value, type.IsArray) + : element == typeof(bool) ? ReadArray(value, type.IsArray) + : element == typeof(int) ? ReadArray(value, type.IsArray) + : element == typeof(long) ? ReadArray(value, type.IsArray) + : element == typeof(float) ? ReadArray(value, type.IsArray) + : element == typeof(double) ? ReadArray(value, type.IsArray) + : throw new NotSupportedException($"Unsupported array element type '{element}'."); + + static object ReadArray(object value, bool array) + => array ? ((IEnumerable)value).ToArray() : ((IEnumerable)value).ToList(); + } + + internal static IReadOnlyList CreateColumns(CollectionModel model, IReadOnlyList> rows) + { + List fields = []; + var key = model.KeyProperty; + fields.Add(key.Type == typeof(long) ? Scalar(key.StorageName) : Scalar(key.StorageName)); + foreach (var property in model.DataProperties) + { + var name = property.StorageName; + if (GetElementType(property.Type) is { } element) + { + fields.Add(element == typeof(string) ? ArrayColumn(name) + : element == typeof(bool) ? ArrayColumn(name) + : element == typeof(int) ? ArrayColumn(name) + : element == typeof(long) ? ArrayColumn(name) + : element == typeof(float) ? ArrayColumn(name) + : element == typeof(double) ? ArrayColumn(name) + : throw new NotSupportedException($"Unsupported array element type '{element}'.")); + } + else + { + var type = property.Type; + fields.Add(type == typeof(string) ? Scalar(name) + : type == typeof(bool) ? Scalar(name) : type == typeof(bool?) ? Scalar(name) + : type == typeof(int) ? Scalar(name) : type == typeof(int?) ? Scalar(name) + : type == typeof(long) ? Scalar(name) : type == typeof(long?) ? Scalar(name) + : type == typeof(float) ? Scalar(name) : type == typeof(float?) ? Scalar(name) + : type == typeof(double) ? Scalar(name) : type == typeof(double?) ? Scalar(name) + : throw new NotSupportedException($"Unsupported Milvus data type '{type}'.")); + } + } + foreach (var property in model.VectorProperties) + { + fields.Add(FieldData.CreateFloatVector(property.StorageName, rows.Select(r => (ReadOnlyMemory)r[property.StorageName]!).ToArray())); + } + return fields; + + FieldData Scalar(string name) => FieldData.Create(name, rows.Select(r => (T)r[name]!).ToArray()); + FieldData ArrayColumn(string name) => FieldData.CreateArray(name, + rows.Select(r => r[name] is { } value ? (IReadOnlyList)((IEnumerable)value).ToArray() : null).ToArray()); + } + + public static object ToKey(object key) => key is Guid guid ? guid.ToString("D") : key; + + // Milvus parses backslash escapes, but does not combine JSON surrogate-pair escapes. + // Keep Unicode code units intact so non-BMP characters reach the server as UTF-8. + internal static string EscapeString(string value) + { + var builder = new StringBuilder(value.Length + 2).Append('"'); + foreach (var c in value) + { + switch (c) + { + case '"': builder.Append("\\\""); break; + case '\\': builder.Append("\\\\"); break; + case '\n': builder.Append("\\n"); break; + case '\r': builder.Append("\\r"); break; + case '\t': builder.Append("\\t"); break; + case '\b': builder.Append("\\b"); break; + case '\f': builder.Append("\\f"); break; + default: + if (c < ' ') + { + builder.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + builder.Append(c); + } + break; + } + } + return builder.Append('"').ToString(); + } + + public static string Literal(object? value) => value switch + { + null => "null", + string text => EscapeString(text), + Guid guid => EscapeString(guid.ToString("D")), + bool boolean => boolean ? "true" : "false", + int or long => Convert.ToString(value, CultureInfo.InvariantCulture)!, + float single when !float.IsNaN(single) && !float.IsInfinity(single) => single.ToString("R", CultureInfo.InvariantCulture), + double number when !double.IsNaN(number) && !double.IsInfinity(number) => number.ToString("R", CultureInfo.InvariantCulture), + _ => throw new NotSupportedException($"Unsupported Milvus filter value '{value.GetType()}'."), + }; +} diff --git a/MEVD/src/Milvus/MilvusFilterTranslator.cs b/MEVD/src/Milvus/MilvusFilterTranslator.cs new file mode 100644 index 0000000..797df79 --- /dev/null +++ b/MEVD/src/Milvus/MilvusFilterTranslator.cs @@ -0,0 +1,201 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Linq.Expressions; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Extensions.VectorData.ProviderServices.Filter; + +namespace CommunityToolkit.VectorData.Milvus; + +internal sealed class MilvusFilterTranslator : FilterTranslatorBase +{ + private string _keyField = null!; + + public string Translate(LambdaExpression filter, CollectionModel model) + { + _keyField = model.KeyProperty.StorageName; + return Translate(PreprocessFilter(filter, model, new FilterPreprocessingOptions()), predicate: true); + } + + // Primary keys are never null. These are valid Milvus predicates, unlike bare true/false. + private string ConstantPredicate(bool value) => $"({_keyField} is {(value ? "not " : "")}null)"; + + private string Translate(Expression expression, bool predicate = false) + { + if (TryBindProperty(expression, out var property)) + { + if (property is VectorPropertyModel) + { + throw new NotSupportedException("Milvus filters cannot reference vector fields."); + } + return predicate && property.Type == typeof(bool) ? $"({property.StorageName} == true)" : property.StorageName; + } + return expression switch + { + ConstantExpression { Value: bool value } when predicate => ConstantPredicate(value), + ConstantExpression constant => MilvusFieldMapping.Literal(constant.Value), + UnaryExpression { NodeType: ExpressionType.Convert } convert + when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type => Translate(convert.Operand, predicate), + UnaryExpression { NodeType: ExpressionType.Not } not => TranslateNegated(not.Operand), + BinaryExpression binary => TranslateBinary(binary), + MethodCallExpression call when TryMatchContains(call, out var source, out var item) + => TryBindProperty(source, out var arrayProperty) && MilvusFieldMapping.GetElementType(arrayProperty.Type) is not null + ? TranslateArrayContains(arrayProperty.StorageName, ConstantValue(item)) + : source switch + { + ConstantExpression { Value: IEnumerable values and not string } => TranslateContains(values, item), + NewArrayExpression { NodeType: ExpressionType.NewArrayInit } array + => TranslateContains(array.Expressions.Select(ConstantValue).ToArray(), item), + _ => throw new NotSupportedException("Milvus Contains requires a constant collection of scalar values."), + }, + MethodCallExpression { Method.Name: nameof(Enumerable.Any), Arguments: [var source, LambdaExpression lambda] } + when lambda.Body is MethodCallExpression call && TryMatchContains(call, out var values, out var item) + && item == lambda.Parameters[0] && TryBindProperty(source, out var anyProperty) + && MilvusFieldMapping.GetElementType(anyProperty.Type) is not null + => TranslateAny(anyProperty.StorageName, values), + _ => throw new NotSupportedException($"Unsupported Milvus filter expression: {expression.NodeType}."), + }; + } + + private string TranslateNegated(Expression expression) + => expression switch + { + UnaryExpression { NodeType: ExpressionType.Not } not => Translate(not.Operand, predicate: true), + BinaryExpression { NodeType: ExpressionType.AndAlso } and => $"({TranslateNegated(and.Left)} or {TranslateNegated(and.Right)})", + BinaryExpression { NodeType: ExpressionType.OrElse } or => $"({TranslateNegated(or.Left)} and {TranslateNegated(or.Right)})", + MethodCallExpression call when TryMatchContains(call, out var source, out var item) + && TryBindProperty(source, out var property) && MilvusFieldMapping.GetElementType(property.Type) is not null + => TranslateArrayContains(property.StorageName, ConstantValue(item), negated: true), + MethodCallExpression { Method.Name: nameof(Enumerable.Any), Arguments: [var source, LambdaExpression lambda] } + when lambda.Body is MethodCallExpression call && TryMatchContains(call, out var values, out var item) + && item == lambda.Parameters[0] && TryBindProperty(source, out var property) + && MilvusFieldMapping.GetElementType(property.Type) is not null + => TranslateAny(property.StorageName, values, negated: true), + _ => $"(not {Translate(expression, predicate: true)})", + }; + + private string TranslateArrayContains(string name, object? value, bool negated = false) + => value is null ? ConstantPredicate(negated) + : GuardArrayMembership(name, $"array_contains({name}, {MilvusFieldMapping.Literal(value)})", negated); + + private static string GuardArrayMembership(string name, string membership, bool negated) + // Milvus 2.5.27 can retain an unknown result under NOT even with an AND null guard. + // An explicit null arm keeps the negated predicate true for null arrays. + => negated ? $"({name} is null or not {membership})" : $"({name} is not null and {membership})"; + + private string TranslateAny(string name, Expression values, bool negated = false) + { + var elements = values switch + { + ConstantExpression { Value: IEnumerable enumerable } => enumerable.Cast(), + NewArrayExpression array => array.Expressions.Select(ConstantValue), + _ => throw new NotSupportedException("Milvus array membership requires constant values."), + }; + var literals = elements.Where(value => value is not null).Select(MilvusFieldMapping.Literal).ToArray(); + return literals.Length == 0 ? ConstantPredicate(negated) + : GuardArrayMembership(name, $"array_contains_any({name}, [{string.Join(", ", literals)}])", negated); + } + + private static object? ConstantValue(Expression expression) => expression switch + { + ConstantExpression constant => constant.Value, + UnaryExpression { NodeType: ExpressionType.Convert } convert + when Nullable.GetUnderlyingType(convert.Type) == convert.Operand.Type => ConstantValue(convert.Operand), + _ => throw new NotSupportedException("Milvus inline filter arrays require constant values."), + }; + + private string TranslateContains(IEnumerable values, Expression item) + { + if (!TryBindProperty(item, out var property) || property is VectorPropertyModel) + { + throw new NotSupportedException("Milvus Contains must test a scalar property."); + } + var elements = values.Cast().ToArray(); + var literals = elements.Where(value => value is not null).Select(MilvusFieldMapping.Literal).ToArray(); + var membership = literals.Length == 0 ? ConstantPredicate(false) : $"({Translate(item)} in [{string.Join(", ", literals)}])"; + if (!property.Type.IsValueType || Nullable.GetUnderlyingType(property.Type) is not null) + { + return elements.Contains(null) + ? $"({property.StorageName} is null or {membership})" + : literals.Length == 0 ? ConstantPredicate(false) : $"({property.StorageName} is not null and {membership})"; + } + return membership; + } + + private static bool IsNull(Expression expression) => expression switch + { + ConstantExpression { Value: null } => true, + UnaryExpression { NodeType: ExpressionType.Convert } convert => IsNull(convert.Operand), + _ => false, + }; + + private string TranslateBinary(BinaryExpression binary) + { + var op = binary.NodeType switch + { + ExpressionType.Equal => "==", ExpressionType.NotEqual => "!=", + ExpressionType.GreaterThan => ">", ExpressionType.GreaterThanOrEqual => ">=", + ExpressionType.LessThan => "<", ExpressionType.LessThanOrEqual => "<=", + ExpressionType.AndAlso => "and", ExpressionType.OrElse => "or", + _ => throw new NotSupportedException($"Unsupported Milvus filter operator: {binary.NodeType}."), + }; + if (binary.NodeType is ExpressionType.Equal or ExpressionType.NotEqual) + { + if (binary.Right is ConstantExpression { Value: null }) + { + return $"({Translate(binary.Left)} is {(binary.NodeType == ExpressionType.NotEqual ? "not " : "")}null)"; + } + if (binary.Left is ConstantExpression { Value: null }) + { + return $"({Translate(binary.Right)} is {(binary.NodeType == ExpressionType.NotEqual ? "not " : "")}null)"; + } + } + if (binary.NodeType is ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + && (IsNull(binary.Left) || IsNull(binary.Right))) + { + // Lifted relational operators are false if either operand is null, including + // captured nullable variables after filter preprocessing. + return ConstantPredicate(false); + } + var logical = binary.NodeType is ExpressionType.AndAlso or ExpressionType.OrElse; + var translated = $"({Translate(binary.Left, logical)} {op} {Translate(binary.Right, logical)})"; + if (binary.NodeType is ExpressionType.AndAlso or ExpressionType.OrElse) + { + return translated; + } + // Milvus uses three-valued null logic. Make comparisons total so C# negation and != + // also match rows whose nullable scalar field is null. + if (TryBindProperty(binary.Left, out var leftField) && IsNullable(leftField.Type) + && TryBindProperty(binary.Right, out var rightField) && IsNullable(rightField.Type)) + { + // Two nullable fields: in C#, == matches two nulls, != matches one null, and a relational operator is false with a null. + var (a, b) = (leftField.StorageName, rightField.StorageName); + var bothNotNull = $"{a} is not null and {b} is not null and {translated}"; + return binary.NodeType switch + { + ExpressionType.Equal => $"(({a} is null and {b} is null) or ({bothNotNull}))", + ExpressionType.NotEqual => $"(({a} is null and {b} is not null) or ({a} is not null and {b} is null) or ({bothNotNull}))", + _ => $"({bothNotNull})", + }; + } + PropertyModel? nullableProperty = null; + if (TryBindProperty(binary.Left, out var left) && IsNullable(left.Type) + && binary.Right is ConstantExpression or UnaryExpression { NodeType: ExpressionType.Convert }) + { + nullableProperty = left; + } + else if (TryBindProperty(binary.Right, out var right) && IsNullable(right.Type) + && binary.Left is ConstantExpression or UnaryExpression { NodeType: ExpressionType.Convert }) + { + nullableProperty = right; + } + return nullableProperty is null ? translated + : binary.NodeType == ExpressionType.NotEqual + ? $"({nullableProperty.StorageName} is null or {translated})" + : $"({nullableProperty.StorageName} is not null and {translated})"; + + static bool IsNullable(Type type) => !type.IsValueType || Nullable.GetUnderlyingType(type) is not null; + } +} diff --git a/MEVD/src/Milvus/MilvusMapper.cs b/MEVD/src/Milvus/MilvusMapper.cs new file mode 100644 index 0000000..9e7e83c --- /dev/null +++ b/MEVD/src/Milvus/MilvusMapper.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Milvus; + +internal sealed class MilvusMapper(CollectionModel model) where TRecord : class +{ + public IDictionary MapFromDataToStorageModel(TRecord record, int recordIndex, + GeneratedEmbeddings>?[]? generatedEmbeddings) + { + var key = model.KeyProperty.GetValueAsObject(record) + ?? throw new InvalidOperationException("The key cannot be null."); + Dictionary row = new(StringComparer.Ordinal) + { + [model.KeyProperty.StorageName] = MilvusFieldMapping.ToKey(key), + }; + foreach (var property in model.DataProperties) + { + var value = property.GetValueAsObject(record); + if (value is IEnumerable elements && MilvusFieldMapping.GetElementType(property.Type) is not null) + { + var elementIndex = 0; + foreach (var element in elements) + { + if (element is null) + { + throw new ArgumentException( + $"Record '{key}' (batch index {recordIndex}) has a null element at index {elementIndex} in array property '{property.ModelName}' (storage field '{property.StorageName}'). Milvus array elements cannot be null.", + nameof(record)); + } + elementIndex++; + } + } + row[property.StorageName] = value; + } + for (var i = 0; i < model.VectorProperties.Count; i++) + { + var property = model.VectorProperties[i]; + var vector = GetVector(generatedEmbeddings?[i] is { } embeddings + ? embeddings[recordIndex] : property.GetValueAsObject(record)); + if (vector.Length != property.Dimensions) + { + throw new ArgumentException($"Vector '{property.ModelName}' must have {property.Dimensions} dimensions."); + } + row[property.StorageName] = vector; + } + return row; + } + + internal static ReadOnlyMemory GetVector(object? value) => value switch + { + ReadOnlyMemory memory => memory, + float[] array => array, + Embedding embedding => embedding.Vector, + _ => throw new InvalidOperationException("A vector must be a non-null float array, ReadOnlyMemory or Embedding."), + }; + + public TRecord MapFromStorageToDataModel(IReadOnlyDictionary row, bool includeVectors) + { + var record = model.CreateRecord()!; + var key = row[model.KeyProperty.StorageName]!; + model.KeyProperty.SetValueAsObject(record, model.KeyProperty.Type == typeof(Guid) ? Guid.Parse((string)key) : key); + foreach (var property in model.DataProperties) + { + if (row.TryGetValue(property.StorageName, out var value)) + { + property.SetValueAsObject(record, MilvusFieldMapping.ReadValue(value, property.Type)); + } + } + if (includeVectors) + { + foreach (var property in model.VectorProperties) + { + if (row.TryGetValue(property.StorageName, out var value)) + { + var vector = GetVector(value); + var type = Nullable.GetUnderlyingType(property.Type) ?? property.Type; + property.SetValueAsObject(record, type == typeof(float[]) ? vector.ToArray() + : type == typeof(Embedding) ? new Embedding(vector) : (object)vector); + } + } + } + return record; + } +} diff --git a/MEVD/src/Milvus/MilvusModelBuilder.cs b/MEVD/src/Milvus/MilvusModelBuilder.cs new file mode 100644 index 0000000..f13e2b1 --- /dev/null +++ b/MEVD/src/Milvus/MilvusModelBuilder.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; + +namespace CommunityToolkit.VectorData.Milvus; + +internal sealed class MilvusModelBuilder() : CollectionModelBuilder(new() +{ + RequiresAtLeastOneVector = true, + SupportsMultipleVectors = true, +}) +{ + // Reserved tokens from Milvus 2.6's Plan.g4 grammar cannot be unquoted field names. + private static readonly HashSet s_reservedNames = new(StringComparer.OrdinalIgnoreCase) + { + "like", "exists", "text_match", "phrase_match", "random_sample", "interval", "iso", + "and", "or", "not", "in", "true", "false", "null", "is", + "json_contains", "json_contains_all", "json_contains_any", + "array_contains", "array_contains_all", "array_contains_any", "array_length", + "st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains", "st_intersects", "st_within", "st_dwithin", + }; + + internal const string SupportedVectorTypes = "ReadOnlyMemory, Embedding, float[]"; + + private const int MaxFieldNameLength = 255; + + protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) + { + base.ValidateKeyProperty(keyProperty); + if (keyProperty.Type != typeof(long) && keyProperty.Type != typeof(string) && keyProperty.Type != typeof(Guid)) + { + throw new NotSupportedException("Milvus keys must be long, string or Guid."); + } + if (keyProperty.IsAutoGenerated && keyProperty.Type != typeof(Guid)) + { + throw new NotSupportedException("Auto-generation is supported only for Guid keys."); + } + } + + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + var name = propertyModel.StorageName; + if (name.Length == 0 || name.Length > MaxFieldNameLength || !(IsLetter(name[0]) || name[0] == '_') + || name.Any(c => !(IsLetter(c) || c is >= '0' and <= '9' || c == '_'))) + { + throw new NotSupportedException($"Invalid Milvus field name '{name}'. Use ASCII letters, digits and underscores, starting with a letter or underscore."); + } + if (s_reservedNames.Contains(name)) + { + throw new NotSupportedException($"Milvus field name '{name}' is a reserved keyword. Configure a different StorageName."); + } + if (propertyModel is VectorPropertyModel vector) + { + _ = MilvusCollectionCreateMapping.GetMetric(vector); + _ = MilvusCollectionCreateMapping.GetIndexType(vector); + } + if (propertyModel is DataPropertyModel { IsFullTextIndexed: true } dataProperty) + { + if (dataProperty.Type != typeof(string)) + { + throw new InvalidOperationException($"Property {nameof(VectorStoreDataProperty.IsFullTextIndexed)} on {nameof(VectorStoreDataProperty)} '{dataProperty.ModelName}' is set to true, but the property type is not a string. The Milvus VectorStore supports {nameof(VectorStoreDataProperty.IsFullTextIndexed)} on string properties only."); + } + var bm25FieldName = MilvusCollectionCreateMapping.GetBm25FieldName(dataProperty); + if (bm25FieldName.Length > MaxFieldNameLength) + { + throw new NotSupportedException($"Milvus field name '{name}' is too long for full-text indexing: its BM25 field '{bm25FieldName}' would exceed {MaxFieldNameLength} characters. Configure a shorter StorageName."); + } + } + + static bool IsLetter(char c) => c is >= 'a' and <= 'z' or >= 'A' and <= 'Z'; + } + + protected override void Validate(Type? type, VectorStoreCollectionDefinition? definition) + { + base.Validate(type, definition); + + // Each full-text indexed property adds a hidden BM25 field, which no property can be stored under. + foreach (var fullTextProperty in DataProperties.Where(p => p.IsFullTextIndexed)) + { + var bm25FieldName = MilvusCollectionCreateMapping.GetBm25FieldName(fullTextProperty); + if (Properties.FirstOrDefault(p => p.StorageName == bm25FieldName) is { } conflict) + { + throw new InvalidOperationException($"Property '{conflict.ModelName}' is mapped to storage name '{bm25FieldName}', which the Milvus VectorStore uses for the BM25 field of the full-text indexed property '{fullTextProperty.ModelName}'. Configure a different StorageName."); + } + } + } + + protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = "string, int, long, float, double, bool, nullable versions of these value types, and arrays/lists of non-nullable scalars"; + if (MilvusFieldMapping.GetElementType(type) is { } elementType) + { + type = elementType; + } + else + { + type = Nullable.GetUnderlyingType(type) ?? type; + } + return type == typeof(string) || type == typeof(int) || type == typeof(long) + || type == typeof(float) || type == typeof(double) || type == typeof(bool); + } + + protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) + => IsVectorPropertyTypeValidCore(type, out supportedTypes); + + internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) + { + supportedTypes = SupportedVectorTypes; + return type == typeof(ReadOnlyMemory) || type == typeof(ReadOnlyMemory?) + || type == typeof(Embedding) || type == typeof(float[]); + } +} diff --git a/MEVD/src/Milvus/MilvusServiceCollectionExtensions.cs b/MEVD/src/Milvus/MilvusServiceCollectionExtensions.cs new file mode 100644 index 0000000..61b05c1 --- /dev/null +++ b/MEVD/src/Milvus/MilvusServiceCollectionExtensions.cs @@ -0,0 +1,328 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using CommunityToolkit.VectorData.Milvus; +using Milvus.Client; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// +/// Extension methods to register and instances on an . +/// +public static class MilvusServiceCollectionExtensions +{ + private const string DynamicCodeMessage = "This method is incompatible with NativeAOT, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + private const string UnreferencedCodeMessage = "This method is incompatible with trimming, consult the documentation for adding collections in a way that's compatible with NativeAOT."; + + /// + /// Registers a as + /// with returned by + /// or retrieved from the dependency injection container if was not provided. + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddMilvusVectorStore( + this IServiceCollection services, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedMilvusVectorStore(services, serviceKey: null, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedMilvusVectorStore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedMilvusVectorStoreCore(services, serviceKey, clientProvider, optionsProvider, lifetime, ownsClient: false); + + private static IServiceCollection AddKeyedMilvusVectorStoreCore( + this IServiceCollection services, + object? serviceKey, + Func? clientProvider, + Func? optionsProvider, + ServiceLifetime lifetime, bool ownsClient, string? databaseName = null) + { + Throw.IfNull(services); + + services.Add(new ServiceDescriptor(typeof(MilvusVectorStore), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetStoreOptions(sp, optionsProvider); + + // Clients created by host overloads are owned, and their database is known; supplied clients remain caller-owned. + return new MilvusVectorStore(new SharedMilvusClient(client, ownsClient, databaseName), options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStore), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , , + /// and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddMilvusVectorStore( + this IServiceCollection services, + string host, + int port = 19530, + bool https = false, + string? apiKey = default, + string? database = null, + string? username = null, + string? password = null, + MilvusVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedMilvusVectorStore(services, serviceKey: null, host, port, https, apiKey, database, username, password, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , , + /// and . + /// + /// The to register the on. + /// The key with which to associate the vector store. + /// The host to connect to. + /// The port to connect to. Defaults to 19530. + /// Whether to encrypt the connection using HTTPS. Defaults to false. + /// The API key to use. Cannot be combined with username/password credentials. + /// The database to connect to, or null for the default database. + /// The authentication username. + /// The authentication password. Required with username. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedMilvusVectorStore( + this IServiceCollection services, + object? serviceKey, + string host, + int port = 19530, + bool https = false, + string? apiKey = default, + string? database = null, + string? username = null, + string? password = null, + MilvusVectorStoreOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + Throw.IfNullOrWhitespace(host); + + ValidateCredentials(apiKey, username, password); + return AddKeyedMilvusVectorStoreCore(services, serviceKey, _ => CreateClient(host, port, https, apiKey, database, username, password), _ => options, + lifetime, ownsClient: true, databaseName: database ?? "default"); + } + + /// + /// Registers a as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddMilvusCollection( + this IServiceCollection services, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedMilvusCollection(services, serviceKey: null, name, clientProvider, optionsProvider, lifetime); + + /// + /// Registers a keyed as + /// with returned by or retrieved from the dependency injection container if was not provided. + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The provider. + /// Options provider to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedMilvusCollection( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider = default, + Func? optionsProvider = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedMilvusCollectionCore(services, serviceKey, name, clientProvider, optionsProvider, lifetime, ownsClient: false); + + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + private static IServiceCollection AddKeyedMilvusCollectionCore( + this IServiceCollection services, + object? serviceKey, + string name, + Func? clientProvider, + Func? optionsProvider, + ServiceLifetime lifetime, bool ownsClient, string? databaseName = null) + where TKey : notnull + where TRecord : class + { + Throw.IfNull(services); + Throw.IfNullOrWhitespace(name); + + services.Add(new ServiceDescriptor(typeof(MilvusCollection), serviceKey, (sp, _) => + { + var client = clientProvider is null ? sp.GetRequiredService() : clientProvider(sp); + var options = GetCollectionOptions(sp, optionsProvider); + + // Clients created by host overloads are owned, and their database is known; supplied clients remain caller-owned. + return new MilvusCollection(() => new SharedMilvusClient(client, ownsClient, databaseName), name, options); + }, lifetime)); + + services.Add(new ServiceDescriptor(typeof(VectorStoreCollection), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IVectorSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + services.Add(new ServiceDescriptor(typeof(IKeywordHybridSearchable), serviceKey, + static (sp, key) => sp.GetRequiredKeyedService>(key), lifetime)); + + return services; + } + + /// + /// Registers a as + /// with created with , , + /// and . + /// + /// + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddMilvusCollection( + this IServiceCollection services, + string name, + string host, + int port = 19530, + bool https = false, + string? apiKey = default, + string? database = null, + string? username = null, + string? password = null, + MilvusCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + => AddKeyedMilvusCollection(services, serviceKey: null, name, host, port, https, apiKey, database, username, password, options, lifetime); + + /// + /// Registers a keyed as + /// with created with , , + /// and . + /// + /// The to register the on. + /// The key with which to associate the collection. + /// The name of the collection. + /// The host to connect to. + /// The port to connect to. Defaults to 19530. + /// Whether to encrypt the connection using HTTPS. Defaults to false. + /// The API key to use. Cannot be combined with username/password credentials. + /// The database to connect to, or null for the default database. + /// The authentication username. + /// The authentication password. Required with username. + /// Options to further configure the . + /// The service lifetime for the store. Defaults to . + /// Service collection. + [RequiresUnreferencedCode(UnreferencedCodeMessage)] + [RequiresDynamicCode(DynamicCodeMessage)] + public static IServiceCollection AddKeyedMilvusCollection( + this IServiceCollection services, + object? serviceKey, + string name, + string host, + int port = 19530, + bool https = false, + string? apiKey = default, + string? database = null, + string? username = null, + string? password = null, + MilvusCollectionOptions? options = default, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + where TKey : notnull + where TRecord : class + { + Throw.IfNullOrWhitespace(host); + + ValidateCredentials(apiKey, username, password); + return AddKeyedMilvusCollectionCore(services, serviceKey, name, _ => CreateClient(host, port, https, apiKey, database, username, password), _ => options, + lifetime, ownsClient: true, databaseName: database ?? "default"); + } + + private static void ValidateCredentials(string? apiKey, string? username, string? password) + { + if ((username is null) != (password is null)) + { + throw new ArgumentException("Username and password must be provided together."); + } + if (apiKey is not null && username is not null) + { + throw new ArgumentException("API key and username/password authentication cannot be combined."); + } + } + + private static MilvusClient CreateClient(string host, int port, bool https, string? apiKey, + string? database, string? username, string? password) + => username is not null ? new MilvusClient(host, username, password!, port, https, database) + : apiKey is not null ? new MilvusClient(host, apiKey, port, https, database) + : new MilvusClient(host, port, https, database); + + private static MilvusVectorStoreOptions? GetStoreOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } + + private static MilvusCollectionOptions? GetCollectionOptions(IServiceProvider sp, Func? optionsProvider) + { + var options = optionsProvider?.Invoke(sp); + if (options?.EmbeddingGenerator is not null) + { + return options; // The user has provided everything, there is nothing to change. + } + + var embeddingGenerator = sp.GetService(); + return embeddingGenerator is null + ? options // There is nothing to change. + : new(options) { EmbeddingGenerator = embeddingGenerator }; // Create a brand new copy in order to avoid modifying the original options. + } +} diff --git a/MEVD/src/Milvus/MilvusVectorStore.cs b/MEVD/src/Milvus/MilvusVectorStore.cs new file mode 100644 index 0000000..5bb2713 --- /dev/null +++ b/MEVD/src/Milvus/MilvusVectorStore.cs @@ -0,0 +1,151 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Milvus.Client; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// Class for accessing the list of collections in a Milvus vector store. +/// +/// +/// This class can be used with collections of any schema type, but requires you to provide schema information when getting a collection. +/// +public sealed class MilvusVectorStore : VectorStore +{ + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + + /// Milvus client that can be used to manage the collections and records in a Milvus store. + private readonly SharedMilvusClient _milvusClient; + + /// A general purpose definition that can be used to construct a collection when needing to proxy schema agnostic operations. + private static readonly VectorStoreCollectionDefinition s_generalPurposeDefinition = new() { Properties = [new VectorStoreKeyProperty("Key", typeof(string)), new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 1)] }; + + private readonly IEmbeddingGenerator? _embeddingGenerator; + + /// Whether the store was disposed: it releases its share of the client only once. + private int _disposed; + + /// + /// Initializes a new instance of the class. + /// + /// Milvus client that can be used to manage the collections and records in a Milvus store. + /// A value indicating whether is disposed after the vector store and the collections it returns are all disposed. + /// Optional configuration options for this class. + public MilvusVectorStore(MilvusClient milvusClient, bool ownsClient, MilvusVectorStoreOptions? options = default) + : this(new SharedMilvusClient(milvusClient, ownsClient), options) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Milvus client that can be used to manage the collections and records in a Milvus store. + /// Optional configuration options for this class. + internal MilvusVectorStore(SharedMilvusClient milvusClient, MilvusVectorStoreOptions? options = default) + { + Throw.IfNull(milvusClient); + + _milvusClient = milvusClient; + + options ??= MilvusVectorStoreOptions.Default; + _embeddingGenerator = options.EmbeddingGenerator; + + _metadata = new() + { + VectorStoreSystemName = MilvusConstants.VectorStoreSystemName, + VectorStoreName = milvusClient.DatabaseName + }; + } + + /// + protected override void Dispose(bool disposing) + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + _milvusClient.Dispose(); + } + + base.Dispose(disposing); + } + +#pragma warning disable IDE0090 // Use 'new(...)' + /// + [RequiresDynamicCode("This overload of GetCollection() is incompatible with NativeAOT. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] + [RequiresUnreferencedCode("This overload of GetCollection() is incompatible with trimming. For dynamic mapping via Dictionary, call GetDynamicCollection() instead.")] +#if NET + public override MilvusCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#else + public override VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) +#endif + => typeof(TRecord) == typeof(Dictionary) + ? throw new ArgumentException(VectorDataStrings.GetCollectionWithDictionaryNotSupported) + : new MilvusCollection(_milvusClient.Share, name, new() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); + + /// +#if NET + public override MilvusDynamicCollection GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#else + public override VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition) +#endif + => new MilvusDynamicCollection(_milvusClient.Share, name, new MilvusCollectionOptions() + { + Definition = definition, + EmbeddingGenerator = _embeddingGenerator + }); +#pragma warning restore IDE0090 + + /// + public override async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + const string OperationName = "ListCollectionNames"; + + var collections = await MilvusErrorHandler.RunAsync>( + new VectorStoreCollectionMetadata { VectorStoreSystemName = MilvusConstants.VectorStoreSystemName, VectorStoreName = _milvusClient.DatabaseName }, + OperationName, + () => _milvusClient.Client.ListCollectionsAsync(cancellationToken: cancellationToken)).ConfigureAwait(false); + + foreach (var collection in collections) + { + yield return collection.Name; + } + } + + /// + public override async Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default) + { + using var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + return await collection.CollectionExistsAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public override async Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default) + { + using var collection = GetDynamicCollection(name, s_generalPurposeDefinition); + await collection.EnsureCollectionDeletedAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? _metadata : + serviceType == typeof(MilvusClient) ? _milvusClient.Client : + serviceType.IsInstanceOfType(this) ? this : + null; + } +} diff --git a/MEVD/src/Milvus/MilvusVectorStoreOptions.cs b/MEVD/src/Milvus/MilvusVectorStoreOptions.cs new file mode 100644 index 0000000..e71571d --- /dev/null +++ b/MEVD/src/Milvus/MilvusVectorStoreOptions.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// Options when creating a . +/// +public sealed class MilvusVectorStoreOptions +{ + internal static readonly MilvusVectorStoreOptions Default = new(); + + /// + /// Initializes a new instance of the class. + /// + public MilvusVectorStoreOptions() + { + } + + internal MilvusVectorStoreOptions(MilvusVectorStoreOptions? source) + { + EmbeddingGenerator = source?.EmbeddingGenerator; + } + + + /// + /// Gets or sets the default embedding generator to use when generating vector embeddings with this vector store. + /// + public IEmbeddingGenerator? EmbeddingGenerator { get; set; } +} diff --git a/MEVD/src/Milvus/README.md b/MEVD/src/Milvus/README.md new file mode 100644 index 0000000..cb370b5 --- /dev/null +++ b/MEVD/src/Milvus/README.md @@ -0,0 +1,63 @@ +# CommunityToolkit.VectorData.Milvus + +Milvus provider for [Microsoft.Extensions.VectorData](https://learn.microsoft.com/dotnet/ai/vector-stores/overview), by the .NET Community Toolkit. + +[Milvus](https://milvus.io/) is an open-source vector database for AI applications. + +The provider is built on [Milvus.Client](https://www.nuget.org/packages/Milvus.Client), the Milvus C# SDK. + +Requires Milvus 2.6 or later. + +## Quick start + +1. Run Milvus with Docker, as a single container with embedded etcd and local storage: + +```bash +docker run -d --name milvus -p 19530:19530 -e DEPLOY_MODE=STANDALONE -e ETCD_USE_EMBED=true -e COMMON_STORAGETYPE=local milvusdb/milvus:v2.6.25 milvus run standalone +``` + +For other setups, see the [standalone installation instructions](https://milvus.io/docs/install_standalone-docker.md). + +2. Install the NuGet package: + +```bash +dotnet add package CommunityToolkit.VectorData.Milvus +``` + +For more information, see the [Microsoft.Extensions.VectorData documentation](https://learn.microsoft.com/dotnet/ai/vector-stores/overview). + +## Limitations + +- Milvus 2.5 is not supported: on Milvus 2.5.27, an upsert of a record with a string property fails with `the length of valid_data of field(...) is wrong`. +- Keys can be `long`, `string`, or `Guid`. Auto-generation is supported only for Guid keys and happens in the provider before upsert. +- Whole-array equality filters, date/time data, and ordered retrieval are not supported. Array membership via `Contains` and `Any` is supported. +- `IsFullTextIndexed` is supported on string properties. Milvus analyzes the text with its default analyzer, and a BM25 function stores the result in a hidden sparse field: the storage name of the property with `_bm25`. No other property can use that name, so the storage name of a full-text indexed property can have at most 250 characters. Records read back do not include the hidden field. +- Keyword hybrid search fuses a vector search and a BM25 search of the keywords on a full-text indexed property, both filtered, with Reciprocal Rank Fusion (k = 60). The keywords are searched as one text, analyzed like the property. Each of the two searches ranks at least 10 records, or `top` + `Skip` when more. Scores are RRF scores: a higher score ranks higher, and a score threshold is a lower bound. `GetService` returns `IKeywordHybridSearchable` only for a collection with a full-text indexed property. +- An array or list property can be null or empty, but its individual elements cannot be null. The provider rejects null elements before calling the SDK and identifies the record, property and element index. +- Supported metrics are cosine similarity (default), dot product similarity, Euclidean distance, and squared Euclidean distance. Scores are native Milvus scores, except Euclidean distance: the square root of the squared distance that Milvus returns. Distance thresholds are upper bounds, similarity thresholds are lower bounds. +- `IsIndexed` does not create scalar indexes; Milvus can filter scalar fields without them. Supported vector indexes are automatic (default), Flat, and HNSW. +- Field storage names must use ASCII letters, digits and underscores, starting with a letter or underscore. Reserved Milvus keywords are rejected during model construction. Varchar fields allow up to 65535 bytes. Schema changes on an existing collection require explicit management through the SDK. +- Collection names must use ASCII letters, digits and underscores, start with a letter or underscore, and have at most 255 characters: Milvus rejects other names, such as `chathistory-local`. +- For NativeAOT, use dynamic dictionary collections with explicit definitions: the typed collection APIs use reflection. Publishing with trimming or NativeAOT shows the warnings `IL2104` and `IL3053` of the dependency `Milvus.Client`. +- On .NET Framework, `Milvus.Client` calls Milvus over WinHTTP (`WinHttpHandler`, which `Grpc.Net.Client` brings in). This needs: + - A TLS endpoint: `https: true` on the host-based registrations, or `ssl: true` on `MilvusClient`. A plaintext endpoint fails with `Response protocol downgraded to HTTP/1.1`. + - A server certificate that Windows trusts. To validate it another way, create the `MilvusClient` on a `GrpcChannel` with a `WinHttpHandler` and pass that client to the provider. + - Windows 11 or later. Windows Server 2019 and 2022 support gRPC over WinHTTP only partially. See [Microsoft's gRPC requirements for .NET Framework](https://learn.microsoft.com/aspnet/core/grpc/netstandard#net-framework). + + Verified on Windows 11 with .NET Framework 4.7.2 against Milvus 2.6.25 over TLS. + +Host-based registrations accept `database`, `username`/`password`, or `apiKey`. The two authentication modes are mutually exclusive. Keyed variants are also available: + +```csharp +builder.Services.AddMilvusVectorStore("milvus.example.com", https: true, + database: "catalog", username: "root", password: password); +``` + +Host-based registrations report the selected database as `VectorStoreName` in the metadata of the store and of its collections, or `default` when none is selected. An existing `MilvusClient` does not expose the database it uses, so with a supplied client `VectorStoreName` is null. Host-based registrations own their SDK client; supplied clients remain caller-owned: + +```csharp +var client = new MilvusClient("milvus.example.com", port: 19530, ssl: true, + database: "catalog"); + +builder.Services.AddMilvusVectorStore(_ => client); +``` diff --git a/MEVD/src/Milvus/SharedMilvusClient.cs b/MEVD/src/Milvus/SharedMilvusClient.cs new file mode 100644 index 0000000..b4a1347 --- /dev/null +++ b/MEVD/src/Milvus/SharedMilvusClient.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.Client; +using Microsoft.Shared.Diagnostics; + +namespace CommunityToolkit.VectorData.Milvus; + +/// +/// The of a vector store or of a collection. A vector store shares it with the collections it returns, +/// and the it owns is disposed when the last of them is disposed. +/// +internal sealed class SharedMilvusClient : IDisposable +{ + private readonly MilvusClient? _ownedClient; + private int _referenceCount = 1; + + /// + /// Initializes a new instance of the class. + /// + /// Milvus client that can be used to manage the collections and records in a Milvus store. + /// A value indicating whether is disposed with the last of the vector store and its collections. + /// + /// The database the SDK client connects to, for telemetry: known to the host-based registrations, which create the client. + /// + public SharedMilvusClient(MilvusClient milvusClient, bool ownsClient, string? databaseName = null) + { + Throw.IfNull(milvusClient); + + Client = milvusClient; + DatabaseName = databaseName; + _ownedClient = ownsClient ? milvusClient : null; + } + + /// + /// Gets the client. + /// + public MilvusClient Client { get; } + + /// + /// Gets the database name for telemetry, or for a client created by the caller: the SDK does not + /// expose the configured database. + /// + public string? DatabaseName { get; } + + /// + /// Gets this client for one more user, a collection of the vector store, which disposes it too. + /// + public SharedMilvusClient Share() + { + if (_ownedClient is not null) + { + Interlocked.Increment(ref _referenceCount); + } + + return this; + } + + /// + public void Dispose() + { + if (_ownedClient is not null && Interlocked.Decrement(ref _referenceCount) == 0) + { + _ownedClient.Dispose(); + } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/Milvus.ConformanceTests.csproj b/MEVD/test/Milvus.ConformanceTests/Milvus.ConformanceTests.csproj new file mode 100644 index 0000000..bf3c562 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/Milvus.ConformanceTests.csproj @@ -0,0 +1,25 @@ + + + + + + net10.0 + enable + enable + true + false + Milvus.ConformanceTests + + + + + + + + + diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusArrayFilterTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusArrayFilterTests.cs new file mode 100644 index 0000000..5caa793 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusArrayFilterTests.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq.Expressions; +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.VectorData; +using Milvus.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests; + +public sealed class MilvusArrayFilterTests : IClassFixture +{ + [Fact] + public async Task ArrayMembershipPreservesNullSemanticsForRetrievalAndSearch() + { + using var collection = new MilvusCollection(MilvusTestStore.Instance.Client, + "ct_arrays_" + Guid.NewGuid().ToString("N"), ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new[] + { + new Record { Id = "null" }, + new Record { Id = "empty", Tags = [], ListTags = [] }, + new Record { Id = "other", Tags = ["y"], ListTags = ["y"] }, + new Record { Id = "match", Tags = ["x"], ListTags = ["x"] }, + }); + var candidates = new[] { "x", null }; + var nulls = new string?[] { null }; + var empty = Array.Empty(); + (Expression> Filter, string[] Expected)[] cases = + [ + (r => r.Tags!.Contains("x"), ["match"]), + (r => !r.Tags!.Contains("x"), ["null", "empty", "other"]), + (r => r.ListTags!.Contains("x"), ["match"]), + (r => !r.ListTags!.Contains("x"), ["null", "empty", "other"]), + (r => r.Tags!.Any(t => new[] { "x", null }.Contains(t)), ["match"]), + (r => !r.Tags!.Any(t => candidates.Contains(t)), ["null", "empty", "other"]), + (r => r.ListTags!.Any(t => candidates.Contains(t)), ["match"]), + (r => !r.ListTags!.Any(t => candidates.Contains(t)), ["null", "empty", "other"]), + (r => r.Tags!.Any(t => nulls.Contains(t)), []), + (r => !r.Tags!.Any(t => nulls.Contains(t)), ["null", "empty", "other", "match"]), + (r => r.Tags!.Any(t => empty.Contains(t)), []), + (r => r.Tags!.Contains(null!), []), + (r => !(r.Tags!.Contains("x") && r.ListTags!.Contains("x")), ["null", "empty", "other"]), + (r => !(r.Tags!.Contains("x") || r.ListTags!.Contains("x")), ["null", "empty", "other"]), + (r => !!r.Tags!.Contains("x"), ["match"]), + ]; + foreach (var (filter, expected) in cases) + { + var retrieved = await collection.GetAsync(filter, 10).ToListAsync(); + Assert.Equal(expected.OrderBy(id => id), retrieved.Select(r => r.Id).OrderBy(id => id)); + var searched = await collection.SearchAsync(new float[] { 1, 0, 0 }, 10, + new() { Filter = filter }).ToListAsync(); + Assert.Equal(expected.OrderBy(id => id), searched.Select(r => r.Record.Id).OrderBy(id => id)); + } + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Record + { + [VectorStoreKey] public string Id { get; set; } = string.Empty; + [VectorStoreData(StorageName = "tags")] public string[]? Tags { get; set; } + [VectorStoreData] public List? ListTags { get; set; } + [VectorStoreVector(3, IndexKind = IndexKind.Flat)] public ReadOnlyMemory Vector { get; set; } = new float[] { 1, 0, 0 }; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusCollectionManagementTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusCollectionManagementTests.cs new file mode 100644 index 0000000..17812f1 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusCollectionManagementTests.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests; +using Xunit; + +namespace Milvus.ConformanceTests; + +public class MilvusCollectionManagementTests(MilvusFixture fixture) + : CollectionManagementTests(fixture), IClassFixture +{ +} + diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusDatabaseTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusDatabaseTests.cs new file mode 100644 index 0000000..89dc42f --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusDatabaseTests.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using CommunityToolkit.VectorData.Milvus; +using Milvus.ConformanceTests.Support; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Xunit; +namespace Milvus.ConformanceTests; +public sealed class MilvusDatabaseTests : IClassFixture +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HostRegistrationUsesSelectedDatabase(bool apiKey) + { + var server = MilvusTestStore.Instance; + var database = "ct_db_" + Guid.NewGuid().ToString("N"); + var name = "records_" + Guid.NewGuid().ToString("N"); + await server.Client.CreateDatabaseAsync(database); + try + { + var services = new ServiceCollection().AddMilvusVectorStore(server.Host, server.Port, database: database, + apiKey: apiKey ? "root:Milvus" : null, username: apiKey ? null : "root", password: apiKey ? null : "Milvus"); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + using var collection = store.GetCollection(name); + try + { + await collection.EnsureCollectionExistsAsync(); + Assert.True(await collection.CollectionExistsAsync()); + Assert.False(await server.Client.HasCollectionAsync(name)); + Assert.Equal(database, Assert.IsType(store.GetService(typeof(VectorStoreMetadata))).VectorStoreName); + Assert.Equal(database, Assert.IsType(collection.GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + finally + { + await server.Client.DropDatabaseAsync(database); + } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusDependencyInjectionTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusDependencyInjectionTests.cs new file mode 100644 index 0000000..357c453 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusDependencyInjectionTests.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using CommunityToolkit.VectorData.Milvus; +using Milvus.Client; +using VectorData.ConformanceTests; +using Xunit; + +namespace Milvus.ConformanceTests; + +public class MilvusDependencyInjectionTests + : DependencyInjectionTests.Record>, long, DependencyInjectionTests.Record> +{ + private const string Host = "localhost"; + private const int Port = 8080; + private const string ApiKey = "fakeKey"; + + protected override void PopulateConfiguration(ConfigurationManager configuration, object? serviceKey = null) + => configuration.AddInMemoryCollection( + [ + new(CreateConfigKey("Milvus", serviceKey, "Host"), Host), + new(CreateConfigKey("Milvus", serviceKey, "Port"), Port.ToString()), + new(CreateConfigKey("Milvus", serviceKey, "ApiKey"), ApiKey), + ]); + + private static string HostProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Milvus", serviceKey, "Host")).Value!; + + private static int PortProvider(IServiceProvider sp, object? serviceKey = null) + => int.Parse(sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Milvus", serviceKey, "Port")).Value!); + + private static string ApiKeyProvider(IServiceProvider sp, object? serviceKey = null) + => sp.GetRequiredService().GetRequiredSection(CreateConfigKey("Milvus", serviceKey, "ApiKey")).Value!; + + public override IEnumerable> CollectionDelegates + { + get + { + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new MilvusClient(Host, ApiKey, Port)) + .AddMilvusCollection(name, lifetime: lifetime) + : services + .AddSingleton(sp => new MilvusClient(Host, ApiKey, Port)) + .AddKeyedMilvusCollection(serviceKey, name, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddMilvusCollection( + name, Host, Port, apiKey: ApiKey, lifetime: lifetime) + : services.AddKeyedMilvusCollection( + serviceKey, name, Host, Port, apiKey: ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, name, lifetime) => serviceKey is null + ? services.AddMilvusCollection( + name, sp => new MilvusClient(HostProvider(sp), ApiKeyProvider(sp), PortProvider(sp)), lifetime: lifetime) + : services.AddKeyedMilvusCollection( + serviceKey, name, sp => new MilvusClient(HostProvider(sp, serviceKey), ApiKeyProvider(sp, serviceKey), PortProvider(sp, serviceKey)), lifetime: lifetime); + } + } + + public override IEnumerable> StoreDelegates + { + get + { + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services.AddMilvusVectorStore( + Host, Port, apiKey: ApiKey, lifetime: lifetime) + : services.AddKeyedMilvusVectorStore( + serviceKey, Host, Port, apiKey: ApiKey, lifetime: lifetime); + + yield return (services, serviceKey, lifetime) => serviceKey is null + ? services + .AddSingleton(sp => new MilvusClient(Host, ApiKey, Port)) + .AddMilvusVectorStore(lifetime: lifetime) + : services + .AddSingleton(sp => new MilvusClient(Host, ApiKey, Port)) + .AddKeyedMilvusVectorStore(serviceKey, lifetime: lifetime); + } + } + + [Fact] + public void HostCantBeNullOrEmpty() + { + IServiceCollection services = new ServiceCollection(); + + Assert.Throws(() => services.AddMilvusVectorStore(host: null!)); + Assert.Throws(() => services.AddKeyedMilvusVectorStore(serviceKey: "notNull", host: null!)); + Assert.Throws(() => services.AddMilvusCollection( + name: "notNull", host: null!)); + Assert.Throws(() => services.AddMilvusCollection( + name: "notNull", host: "")); + Assert.Throws(() => services.AddKeyedMilvusCollection( + serviceKey: "notNull", name: "notNull", host: null!)); + Assert.Throws(() => services.AddKeyedMilvusCollection( + serviceKey: "notNull", name: "notNull", host: "")); + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusDistanceFunctionTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusDistanceFunctionTests.cs new file mode 100644 index 0000000..071ea79 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusDistanceFunctionTests.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests; +public class MilvusDistanceFunctionTests(MilvusDistanceFunctionTests.Fixture fixture) : DistanceFunctionTests(fixture), IClassFixture +{ + public override Task NegativeDotProductSimilarity() => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + public override Task ManhattanDistance() => Assert.ThrowsAsync(base.ManhattanDistance); + public override Task HammingDistance() => Assert.ThrowsAsync(base.HammingDistance); + public override Task CosineDistance() => Assert.ThrowsAsync(base.CosineDistance); + public new class Fixture : DistanceFunctionTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusEmbeddingGenerationTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusEmbeddingGenerationTests.cs new file mode 100644 index 0000000..73472e3 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusEmbeddingGenerationTests.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using Milvus.Client; +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests; + +public class MilvusEmbeddingGenerationTests(MilvusEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, MilvusEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture) + : EmbeddingGenerationTests(stringVectorFixture, romOfFloatVectorFixture), IClassFixture, IClassFixture +{ + public new class StringVectorFixture : EmbeddingGenerationTests.StringVectorFixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => MilvusTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(MilvusTestStore.Instance.Client) + .AddMilvusVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(MilvusTestStore.Instance.Client) + .AddMilvusCollection(this.CollectionName) + ]; + } + + public new class RomOfFloatVectorFixture : EmbeddingGenerationTests.RomOfFloatVectorFixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + + public override VectorStore CreateVectorStore(IEmbeddingGenerator? embeddingGenerator) + => MilvusTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = embeddingGenerator }); + + public override Func[] DependencyInjectionStoreRegistrationDelegates => + [ + services => services + .AddSingleton(MilvusTestStore.Instance.Client) + .AddMilvusVectorStore() + ]; + + public override Func[] DependencyInjectionCollectionRegistrationDelegates => + [ + services => services + .AddSingleton(MilvusTestStore.Instance.Client) + .AddMilvusCollection(this.CollectionName) + ]; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusEnsureCollectionTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusEnsureCollectionTests.cs new file mode 100644 index 0000000..a0a4193 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusEnsureCollectionTests.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Milvus.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests; + +public sealed class MilvusEnsureCollectionTests : IClassFixture +{ + [Fact] + public async Task EnsureCollectionExistsKeepsTheIndexOfAnExistingCollection() + { + // The collection exists with an HNSW index, where the model asks for a Flat one: as another tool could create it. + var client = MilvusTestStore.Instance.Client; + var name = "ct_existing_index_" + Guid.NewGuid().ToString("N"); + await client.CreateCollectionAsync(name, + [FieldSchema.CreateVarchar("Id", 65535, isPrimaryKey: true), FieldSchema.CreateVarchar("Text", 65535, nullable: true), FieldSchema.CreateFloatVector("Vector", 3)]); + var records = client.GetCollection(name); + await records.CreateIndexAsync("Vector", IndexType.Hnsw, SimilarityMetricType.Cosine, + extraParams: new Dictionary { ["M"] = "8", ["efConstruction"] = "64" }); + using var collection = new MilvusCollection(client, name, ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new Record { Id = "a", Text = "apples", Vector = new float[] { 1, 0, 0 } }); + + Assert.Equal("a", Assert.Single(await collection.SearchAsync(new float[] { 1, 0, 0 }, 1).ToListAsync()).Record.Id); + Assert.Contains(Assert.Single(await records.DescribeIndexAsync("Vector")).Params, p => p.Value == "HNSW"); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task EnsureCollectionExistsAcceptsAnExistingCollectionWithoutTheBm25Field() + { + // The collection was created before the text property was full-text indexed: it has no BM25 field. + var client = MilvusTestStore.Instance.Client; + var name = "ct_existing_text_" + Guid.NewGuid().ToString("N"); + using (var before = new MilvusCollection(client, name, ownsClient: false)) + { + await before.EnsureCollectionExistsAsync(); + } + using var collection = new MilvusCollection(client, name, ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new FullTextRecord { Id = "a", Text = "apples", Vector = new float[] { 1, 0, 0 } }); + + Assert.Equal("a", Assert.Single(await collection.SearchAsync(new float[] { 1, 0, 0 }, 1).ToListAsync()).Record.Id); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Record + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreData] + public string? Text { get; set; } + + [VectorStoreVector(3, IndexKind = IndexKind.Flat)] + public ReadOnlyMemory Vector { get; set; } + } + + public sealed class FullTextRecord + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreData(IsFullTextIndexed = true)] + public string? Text { get; set; } + + [VectorStoreVector(3, IndexKind = IndexKind.Flat)] + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusFilterTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusFilterTests.cs new file mode 100644 index 0000000..d27add3 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusFilterTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests; +public class MilvusFilterTests(MilvusFilterTests.Fixture fixture) : FilterTests(fixture), IClassFixture +{ + public new class Fixture : FilterTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusFullTextTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusFullTextTests.cs new file mode 100644 index 0000000..a58ea0a --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusFullTextTests.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Milvus.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests; + +public sealed class MilvusFullTextTests : IClassFixture +{ + [Fact] + public async Task ChatHistoryMemoryProviderDefinitionStoresReadsAndSearchesMessages() + { + // The definition ChatHistoryMemoryProvider of Microsoft.Agents.AI 1.23.0 creates, as a dynamic collection. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("Role", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("MessageId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("AuthorName", typeof(string)), + new VectorStoreDataProperty("ApplicationId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("AgentId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("UserId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("SessionId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("Content", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreDataProperty("CreatedAt", typeof(string)) { IsIndexed = true }, + new VectorStoreVectorProperty("ContentEmbedding", typeof(string), 4), + ], + }; + using var store = MilvusTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = new ConstantEmbeddingGenerator() }); + using var collection = store.GetDynamicCollection("ct_chat_history_" + Guid.NewGuid().ToString("N"), definition); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.EnsureCollectionExistsAsync(); + var question = Message("user", "What is the capital of France?"); + var answer = Message("assistant", "The capital of France is Paris."); + var empty = Message("assistant", null); + await collection.UpsertAsync([question, answer, empty]); + + var read = await collection.GetAsync(answer["Key"]!); + Assert.NotNull(read); + Assert.Equal("The capital of France is Paris.", read["Content"]); + Assert.DoesNotContain("Content_bm25", read.Keys); + Assert.Null((await collection.GetAsync(empty["Key"]!))!["Content"]); + + var results = await collection.SearchAsync("capital", 3, new() { Filter = r => (string?)r["Role"] == "user" }).ToListAsync(); + Assert.Equal(question["Key"], Assert.Single(results).Record["Key"]); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + + static Dictionary Message(string role, string? content) => new() + { + ["Key"] = Guid.NewGuid(), + ["Role"] = role, + ["MessageId"] = Guid.NewGuid().ToString(), + ["AuthorName"] = null, + ["ApplicationId"] = "app", + ["AgentId"] = "agent", + ["UserId"] = "user", + ["SessionId"] = "session", + ["Content"] = content, + ["CreatedAt"] = "2026-10-23T10:00:00Z", + ["ContentEmbedding"] = content ?? string.Empty, + }; + } + + [Fact] + public async Task TextSearchStoreSearchesItsDocumentsWithHybridSearch() + { + // TextSearchStore of Microsoft.SemanticKernel.Core 1.80.1 creates a typed collection with this definition, + // and searches it with the query, the words of the query as keywords, and a filter on its namespaces. + using var store = MilvusTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = new ConstantEmbeddingGenerator() }); + using var collection = store.GetCollection("ct_text_search_" + Guid.NewGuid().ToString("N"), TextSearchStoreDocument.Definition(4)); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync( + [ + Document("apples", "Apples are a healthy and nourishing snack", "fruit"), + Document("oranges", "Oranges are tangy and contain vitamin c", "fruit"), + Document("grapes", "Grapes are healthy, sweet and juicy", "fruit"), + Document("grapevine", "A grapevine climbs the wall", "garden"), + Document("unread", null, "fruit"), + ]); + var hybrid = Assert.IsAssignableFrom>( + collection.GetService(typeof(IKeywordHybridSearchable))); + var query = "Which fruit is tangy?"; + var keywords = query.Split([' ', '?'], StringSplitOptions.RemoveEmptyEntries); + var nameSpace = "fruit"; + + var results = await hybrid.HybridSearchAsync(query, keywords, 3, new() { Filter = d => d.Namespaces.Contains(nameSpace) }).ToListAsync(); + + Assert.Equal(3, results.Count); + Assert.Equal("oranges", results[0].Record.Key); + Assert.Equal("Oranges are tangy and contain vitamin c", results[0].Record.Text); + Assert.DoesNotContain(results, r => r.Record.Key == "grapevine"); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + + static TextSearchStoreDocument Document(string key, string? text, string nameSpace) => new() + { + Key = key, + Namespaces = [nameSpace], + Text = text, + SourceId = key, + SourceName = key + ".txt", + SourceLink = "https://example.com/" + key, + TextEmbedding = text ?? string.Empty, + }; + } + + [Fact] + public async Task ChatHistoryMemoryProviderDefinitionSupportsHybridSearch() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("Role", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("Content", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreVectorProperty("ContentEmbedding", typeof(string), 4), + ], + }; + using var store = MilvusTestStore.Instance.GetVectorStore(new() { EmbeddingGenerator = new ConstantEmbeddingGenerator() }); + using var collection = store.GetDynamicCollection("ct_chat_hybrid_" + Guid.NewGuid().ToString("N"), definition); + try + { + await collection.EnsureCollectionExistsAsync(); + var paris = Guid.NewGuid(); + await collection.UpsertAsync( + [ + new() { ["Key"] = Guid.NewGuid(), ["Role"] = "user", ["Content"] = "What is the capital of France?", ["ContentEmbedding"] = "q" }, + new() { ["Key"] = paris, ["Role"] = "assistant", ["Content"] = "Paris is the capital of France.", ["ContentEmbedding"] = "a" }, + new() { ["Key"] = Guid.NewGuid(), ["Role"] = "assistant", ["Content"] = "Rome is the capital of Italy.", ["ContentEmbedding"] = "b" }, + ]); + var hybrid = Assert.IsAssignableFrom>>( + collection.GetService(typeof(IKeywordHybridSearchable>))); + + var results = await hybrid.HybridSearchAsync("capital", ["Paris"], 2, new() { Filter = r => (string?)r["Role"] == "assistant" }).ToListAsync(); + + Assert.Equal(2, results.Count); + Assert.Equal(paris, results[0].Record["Key"]); + Assert.DoesNotContain("Content_bm25", results[0].Record.Keys); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + private sealed class TextSearchStoreDocument + { + public string Key { get; set; } = string.Empty; + + public List Namespaces { get; set; } = []; + + public string? Text { get; set; } + + public string? SourceId { get; set; } + + public string? SourceName { get; set; } + + public string? SourceLink { get; set; } + + public string? TextEmbedding { get; set; } + + public static VectorStoreCollectionDefinition Definition(int dimensions) => new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Namespaces", typeof(List)) { IsIndexed = true }, + new VectorStoreDataProperty("SourceId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreDataProperty("SourceName", typeof(string)), + new VectorStoreDataProperty("SourceLink", typeof(string)), + new VectorStoreVectorProperty("TextEmbedding", typeof(string), dimensions), + ], + }; + } + + private sealed class ConstantEmbeddingGenerator : IEmbeddingGenerator> + { + public Task>> GenerateAsync(IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(new GeneratedEmbeddings>(values.Select(_ => new Embedding(new float[] { 1, 0, 0, 0 })))); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusHybridSearchTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusHybridSearchTests.cs new file mode 100644 index 0000000..d026a46 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusHybridSearchTests.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.VectorData; +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests; +public class MilvusHybridSearchTests(MilvusHybridSearchTests.VectorAndStringFixture single, MilvusHybridSearchTests.MultiTextFixture multi) + : HybridSearchTests(single, multi), IClassFixture, IClassFixture +{ + private static readonly ReadOnlyMemory s_vector = new float[] { 1, 0, 0, 0 }; + + [Fact] + public async Task HybridSearchAsync_with_a_filter_matching_no_record_returns_no_record() + => Assert.Empty(await single.HybridSearchable.HybridSearchAsync(s_vector, ["Oranges"], 3, new() { Filter = r => r.Code == 99 }).ToListAsync()); + + [Fact] + public async Task HybridSearchAsync_with_ScoreThreshold_keeps_the_scores_at_or_above_it() + { + // Only the record with the keyword ranks in both searches: its RRF score is well above the others, whatever the + // order of the records with the same vector. + var all = await single.HybridSearchable.HybridSearchAsync(s_vector, ["Grapes"], 3).ToListAsync(); + var threshold = (all[0].Score!.Value + all[1].Score!.Value) / 2; + + var results = await single.HybridSearchable.HybridSearchAsync(s_vector, ["Grapes"], 3, new() { ScoreThreshold = threshold }).ToListAsync(); + + Assert.Equal(3, Assert.Single(results).Record.Code); + } + + [Fact] + public async Task HybridSearchAsync_with_IncludeVectors_returns_the_vectors() + { + var withVectors = await single.HybridSearchable.HybridSearchAsync(s_vector, ["Grapes"], 1, new() { IncludeVectors = true }).ToListAsync(); + var withoutVectors = await single.HybridSearchable.HybridSearchAsync(s_vector, ["Grapes"], 1).ToListAsync(); + + Assert.Equal(s_vector.ToArray(), Assert.Single(withVectors).Record.Vector.ToArray()); + Assert.True(Assert.Single(withoutVectors).Record.Vector.IsEmpty); + } + + [Fact] + public async Task HybridSearchAsync_with_top_and_Skip_at_the_Milvus_limit() + => Assert.Equal(2, (await single.HybridSearchable.HybridSearchAsync(s_vector, ["Grapes"], 16383, new() { Skip = 1 }).ToListAsync()).Count); + + [Fact] + public async Task HybridSearchAsync_without_keywords_returns_the_vector_ranking() + { + // Distinct vectors, so that the vector ranking is near, middle, far; the keyword would rank far and middle first. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreVectorProperty("Vector", typeof(ReadOnlyMemory), 4) { IndexKind = IndexKind.Flat }, + ], + }; + using var store = MilvusTestStore.Instance.GetVectorStore(new()); + using var collection = store.GetDynamicCollection("ct_hybrid_no_keywords_" + Guid.NewGuid().ToString("N"), definition); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync( + [ + new() { ["Key"] = "near", ["Text"] = "Pears", ["Vector"] = new ReadOnlyMemory([1, 0, 0, 0]) }, + new() { ["Key"] = "middle", ["Text"] = "Apples and apples", ["Vector"] = new ReadOnlyMemory([1, 1, 0, 0]) }, + new() { ["Key"] = "far", ["Text"] = "Apples", ["Vector"] = new ReadOnlyMemory([0, 0, 1, 0]) }, + ]); + var hybrid = (IKeywordHybridSearchable>)collection.GetService(typeof(IKeywordHybridSearchable>))!; + + var vectorRanking = await collection.SearchAsync(s_vector, 3).Select(r => (string)r.Record["Key"]!).ToListAsync(); + var withoutKeywords = await hybrid.HybridSearchAsync(s_vector, [], 3).ToListAsync(); + var withKeyword = await hybrid.HybridSearchAsync(s_vector, ["Apples"], 3).Select(r => (string)r.Record["Key"]!).ToListAsync(); + + Assert.Equal(["near", "middle", "far"], vectorRanking); + Assert.Equal(vectorRanking, withoutKeywords.Select(r => (string)r.Record["Key"]!)); + for (var rank = 0; rank < 3; rank++) + { + // The RRF score of the vector ranking alone, with k = 60. + Assert.Equal(1d / (61 + rank), withoutKeywords[rank].Score!.Value, 1e-6); + } + Assert.Equal("near", withKeyword[2]); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public new class VectorAndStringFixture : HybridSearchTests.VectorAndStringFixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } + public new class MultiTextFixture : HybridSearchTests.MultiTextFixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusIndexKindTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusIndexKindTests.cs new file mode 100644 index 0000000..73e016a --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusIndexKindTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests; +public class MilvusIndexKindTests(MilvusIndexKindTests.Fixture fixture) : IndexKindTests(fixture), IClassFixture +{ + public new class Fixture : IndexKindTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusNullComparisonTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusNullComparisonTests.cs new file mode 100644 index 0000000..74b3d16 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusNullComparisonTests.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System.Linq.Expressions; +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.VectorData; +using Milvus.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests; +public sealed class MilvusNullComparisonTests : IClassFixture +{ + [Fact] + public async Task NullComparisonsMatchCSharpForRetrievalAndSearch() + { + using var collection = new MilvusCollection(MilvusTestStore.Instance.Client, + "ct_null_" + Guid.NewGuid().ToString("N"), ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new[] + { + new MilvusRoundTripTests.Record { Id = "null", Count = null, Vector = new float[] { 1, 0, 0 } }, + new MilvusRoundTripTests.Record { Id = "value", Count = 1, Vector = new float[] { 1, 0, 0 } }, + }); + int? value = null; + Assert.Empty(await collection.GetAsync(r => r.Count > value, 10).ToListAsync()); + Assert.Empty(await collection.GetAsync(r => r.Count >= value, 10).ToListAsync()); + Assert.Empty(await collection.GetAsync(r => value < r.Count, 10).ToListAsync()); + Assert.Empty(await collection.GetAsync(r => value <= r.Count, 10).ToListAsync()); + Assert.Equal(2, (await collection.GetAsync(r => !(r.Count > value), 10).ToListAsync()).Count); + Assert.Empty(await collection.SearchAsync(new float[] { 1, 0, 0 }, 10, + new() { Filter = r => r.Count > value }).ToListAsync()); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task ComparisonsOfTwoPropertiesMatchCSharpWithEveryCombinationOfNulls() + { + using var collection = new MilvusCollection(MilvusTestStore.Instance.Client, + "ct_null_pairs_" + Guid.NewGuid().ToString("N"), ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + int?[] numbers = [null, 1, 2]; + string?[] texts = [null, "x", "y"]; + var records = (from a in Enumerable.Range(0, 3) + from b in Enumerable.Range(0, 3) + select new Pair + { + Id = $"{a}{b}", + A = numbers[a], + B = numbers[b], + I = b, + S = texts[a], + T = texts[b], + Vector = new float[] { 1, 0 }, + }).ToList(); + await collection.UpsertAsync(records); + Expression>[] filters = + [ + r => r.A == r.B, r => r.A != r.B, r => r.A < r.B, r => r.A <= r.B, r => r.A > r.B, r => r.A >= r.B, + r => !(r.A == r.B), r => !(r.A != r.B), r => !(r.A < r.B), r => !(r.A <= r.B), r => !(r.A > r.B), r => !(r.A >= r.B), + r => r.S == r.T, r => r.S != r.T, r => !(r.S == r.T), r => !(r.S != r.T), + r => r.A == r.I, r => r.I != r.A, r => r.I < r.A, r => !(r.A == r.I), r => !(r.I < r.A), + r => r.A == r.B || r.S == r.T, r => !(r.A == r.B && r.S != r.T), + ]; + + foreach (var filter in filters) + { + var expected = records.Where(filter.Compile()).Select(r => r.Id).Order(StringComparer.Ordinal).ToList(); + var retrieved = (await collection.GetAsync(filter, 100).ToListAsync()).Select(r => r.Id).Order(StringComparer.Ordinal).ToList(); + var found = (await collection.SearchAsync(new float[] { 1, 0 }, 100, new() { Filter = filter }).ToListAsync()) + .Select(r => r.Record.Id).Order(StringComparer.Ordinal).ToList(); + Assert.True(expected.SequenceEqual(retrieved), $"{filter}: expected [{string.Join(", ", expected)}], retrieved [{string.Join(", ", retrieved)}]"); + Assert.True(expected.SequenceEqual(found), $"{filter}: expected [{string.Join(", ", expected)}], found [{string.Join(", ", found)}]"); + } + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Pair + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreData] + public int? A { get; set; } + + [VectorStoreData] + public int? B { get; set; } + + [VectorStoreData] + public int I { get; set; } + + [VectorStoreData] + public string? S { get; set; } + + [VectorStoreData] + public string? T { get; set; } + + [VectorStoreVector(2, IndexKind = IndexKind.Flat)] + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusRoundTripTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusRoundTripTests.cs new file mode 100644 index 0000000..16ca41e --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusRoundTripTests.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; +using Milvus.ConformanceTests.Support; + +namespace Milvus.ConformanceTests; + +public sealed class MilvusRoundTripTests : IClassFixture +{ + private readonly MilvusClient _client; + public MilvusRoundTripTests(MilvusFixture fixture) => _client = ((MilvusTestStore)fixture.TestStore).Client; + [Fact] + public async Task TypedCrudFilteringAndSearch() + { + var client = _client; + using var collection = new MilvusCollection(client, "ct_" + Guid.NewGuid().ToString("N"), ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + Assert.True(await collection.CollectionExistsAsync()); + await collection.UpsertAsync(new Record { Id = "a", Label = "first", Count = null, Vector = new float[] { 1, 0, 0 } }); + await collection.UpsertAsync(new Record { Id = "b", Label = "second", Count = 2, Vector = new float[] { 0, 1, 0 } }); + var first = await collection.GetAsync("a", new() { IncludeVectors = true }); + Assert.NotNull(first); + Assert.Null(first.Count); + Assert.Equal("first", first.Label); + Assert.Equal(new float[] { 1, 0, 0 }, first.Vector.ToArray()); + var found = await collection.SearchAsync(new float[] { 1, 0, 0 }, 1).ToListAsync(); + Assert.Equal("a", Assert.Single(found).Record.Id); + Assert.Equal(2, (await collection.GetAsync(r => true, 10).ToListAsync()).Count); + Assert.Empty(await collection.GetAsync(r => false, 10).ToListAsync()); + var noIds = Array.Empty(); + Assert.Empty(await collection.GetAsync(r => noIds.Contains(r.Id), 10).ToListAsync()); + var filtered = await collection.GetAsync(r => r.Count == 2, 10).ToListAsync(); + Assert.Equal("b", Assert.Single(filtered).Id); + await collection.UpsertAsync(new Record { Id = "a", Label = "updated", Count = 3, Vector = new float[] { 1, 0, 0 } }); + Assert.Equal("updated", (await collection.GetAsync("a"))!.Label); + await collection.DeleteAsync("a"); + Assert.Null(await collection.GetAsync("a")); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Fact] + public async Task DynamicMultipleVectorsAndLongKeys() + { + var client = _client; + using var store = new MilvusVectorStore(client, ownsClient: false); + using var collection = store.GetDynamicCollection("ct_" + Guid.NewGuid().ToString("N"), new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(long)), + new VectorStoreDataProperty("Label", typeof(string)), + new VectorStoreVectorProperty("First", typeof(float[]), 3) { IndexKind = IndexKind.Flat }, + new VectorStoreVectorProperty("Second", typeof(float[]), 3) { IndexKind = IndexKind.Flat }], + }); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new Dictionary + { + ["Id"] = 42L, ["Label"] = "two vectors", ["First"] = new float[] { 1, 0, 0 }, ["Second"] = new float[] { 0, 1, 0 }, + }); + var row = await collection.GetAsync(42L, new() { IncludeVectors = true }); + Assert.NotNull(row); + Assert.Equal(42L, row["Id"]); + Assert.Equal(new float[] { 1, 0, 0 }, Assert.IsType(row["First"])); + Assert.Equal(new float[] { 0, 1, 0 }, Assert.IsType(row["Second"])); + var results = await collection.SearchAsync(new float[] { 0, 1, 0 }, 1, + new() { VectorProperty = r => r["Second"] }).ToListAsync(); + Assert.Equal(42L, Assert.Single(results).Record["Id"]); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SearchReturnsNoResultsForUnmatchedFiltersAndDeletedRecords(bool includeVectors) + { + var client = _client; + using var collection = new MilvusCollection(client, "ct_" + Guid.NewGuid().ToString("N"), ownsClient: false); + try + { + await collection.EnsureCollectionExistsAsync(); + await collection.UpsertAsync(new Record { Id = "one", Count = 1, Vector = new float[] { 1, 0, 0 } }); + Assert.Empty(await collection.SearchAsync(new float[] { 1, 0, 0 }, 5, + new() { Filter = r => r.Count == 99, IncludeVectors = includeVectors }).ToListAsync()); + await collection.DeleteAsync("one"); + Assert.Empty(await collection.SearchAsync(new float[] { 1, 0, 0 }, 5, + new() { IncludeVectors = includeVectors }).ToListAsync()); + } + finally + { + await collection.EnsureCollectionDeletedAsync(); + } + } + + public sealed class Record + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + [VectorStoreData] + public string? Label { get; set; } + [VectorStoreData] + public int? Count { get; set; } + [VectorStoreVector(3, IndexKind = IndexKind.Flat)] + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/MilvusTestSuiteImplementationTests.cs b/MEVD/test/Milvus.ConformanceTests/MilvusTestSuiteImplementationTests.cs new file mode 100644 index 0000000..b1f7783 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/MilvusTestSuiteImplementationTests.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests; +namespace Milvus.ConformanceTests; +public class MilvusTestSuiteImplementationTests : TestSuiteImplementationTests +{ +} diff --git a/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusBasicModelTests.cs b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusBasicModelTests.cs new file mode 100644 index 0000000..64514dc --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusBasicModelTests.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests.ModelTests; + +public class MilvusBasicModelTests(MilvusBasicModelTests.Fixture fixture) + : BasicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_OrderBy() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_OrderBy_and_Skip() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy_and_Skip); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public new class Fixture : BasicModelTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusDynamicModelTests.cs b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusDynamicModelTests.cs new file mode 100644 index 0000000..8abdf89 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusDynamicModelTests.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests.ModelTests; + +public class MilvusDynamicModelTests(MilvusDynamicModelTests.Fixture fixture) + : DynamicModelTests(fixture), IClassFixture +{ + public override async Task GetAsync_with_filter_and_OrderBy() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_OrderBy_and_Skip() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_OrderBy_and_Skip); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public override async Task GetAsync_with_filter_and_multiple_OrderBys() + { + var exception = await Assert.ThrowsAsync(base.GetAsync_with_filter_and_multiple_OrderBys); + Assert.Equal("Milvus does not support ordering.", exception.Message); + } + + public new class Fixture : DynamicModelTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusMultiVectorModelTests.cs b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusMultiVectorModelTests.cs new file mode 100644 index 0000000..fe7d86a --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusMultiVectorModelTests.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests.ModelTests; + +public class MilvusMultiVectorModelTests(MilvusMultiVectorModelTests.Fixture fixture) + : MultiVectorModelTests(fixture), IClassFixture +{ + public new class Fixture : MultiVectorModelTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoDataModelTests.cs b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoDataModelTests.cs new file mode 100644 index 0000000..228d044 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoDataModelTests.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; + +namespace Milvus.ConformanceTests.ModelTests; + +public class MilvusNoDataModelTests(MilvusNoDataModelTests.Fixture fixture) + : NoDataModelTests(fixture), IClassFixture +{ + public new class Fixture : NoDataModelTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} + diff --git a/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoVectorModelTests.cs b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoVectorModelTests.cs new file mode 100644 index 0000000..8ae4af7 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/ModelTests/MilvusNoVectorModelTests.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using CommunityToolkit.VectorData.Milvus; +using VectorData.ConformanceTests.ModelTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests.ModelTests; +public class MilvusNoVectorModelTests(MilvusNoVectorModelTests.Fixture fixture) : NoVectorModelTests(fixture), IClassFixture, IAsyncLifetime +{ + ValueTask IAsyncLifetime.InitializeAsync() => default; + ValueTask IAsyncDisposable.DisposeAsync() => default; + public override Task GetAsync_single_record(bool includeVectors) { fixture.AssertUnsupported(); return Task.CompletedTask; } + public override Task Insert_single_record() { fixture.AssertUnsupported(); return Task.CompletedTask; } + public override Task Delete_single_record() { fixture.AssertUnsupported(); return Task.CompletedTask; } + public new class Fixture : NoVectorModelTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + public override async ValueTask InitializeAsync() => await TestStore.ReferenceCountingStartAsync(); + public override async ValueTask DisposeAsync() => await TestStore.ReferenceCountingStopAsync(); + public void AssertUnsupported() + { + using var store = new MilvusVectorStore(MilvusTestStore.Instance.Client, ownsClient: false); + Assert.Throws(() => store.GetDynamicCollection("unsupported", CreateRecordDefinition())); + } + } +} + diff --git a/MEVD/test/Milvus.ConformanceTests/Support/MilvusFixture.cs b/MEVD/test/Milvus.ConformanceTests/Support/MilvusFixture.cs new file mode 100644 index 0000000..e298a90 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/Support/MilvusFixture.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using VectorData.ConformanceTests.Support; + +namespace Milvus.ConformanceTests.Support; + +public sealed class MilvusFixture : VectorStoreFixture +{ + public override TestStore TestStore => MilvusTestStore.Instance; +} diff --git a/MEVD/test/Milvus.ConformanceTests/Support/MilvusTestStore.cs b/MEVD/test/Milvus.ConformanceTests/Support/MilvusTestStore.cs new file mode 100644 index 0000000..43195f5 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/Support/MilvusTestStore.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using CommunityToolkit.VectorData.Milvus; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Testcontainers.Milvus; +using VectorData.ConformanceTests.Support; + +namespace Milvus.ConformanceTests.Support; + +#pragma warning disable CA1001 // The base TestStore controls resource lifetime. +internal sealed class MilvusTestStore : TestStore +{ + public static MilvusTestStore Instance { get; } = new(); + public override string DefaultIndexKind => IndexKind.Flat; + private readonly MilvusContainer? _container = Environment.GetEnvironmentVariable("MILVUS_HOST") is null + ? new MilvusBuilder("milvusdb/milvus:v2.6.25").Build() : null; + public string Host { get; private set; } = null!; + public int Port { get; private set; } + private MilvusClient? _client; + public MilvusClient Client => _client ?? throw new InvalidOperationException("Not initialized"); + public MilvusVectorStore GetVectorStore(MilvusVectorStoreOptions options) => new(Client, ownsClient: false, options); + + private MilvusTestStore() + { + } + + protected override async Task StartAsync() + { + if (_container is not null) + { + await _container.StartAsync(); + } + var host = Environment.GetEnvironmentVariable("MILVUS_HOST") ?? _container!.Hostname; + var port = int.Parse(Environment.GetEnvironmentVariable("MILVUS_PORT") ?? "19530", System.Globalization.CultureInfo.InvariantCulture); + Host = host; + Port = _container?.GetMappedPublicPort(19530) ?? port; + _client = new MilvusClient(Host, Port); + DefaultVectorStore = new MilvusVectorStore(_client, ownsClient: false); + } + + protected override async Task StopAsync() + { + DefaultVectorStore.Dispose(); + _client?.Dispose(); + if (_container is not null) + { + await _container.DisposeAsync(); + } + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusDataTypeTests.cs b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusDataTypeTests.cs new file mode 100644 index 0000000..1cb875a --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusDataTypeTests.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests.TypeTests; +public class MilvusDataTypeTests(MilvusDataTypeTests.Fixture fixture) : DataTypeTests.DefaultRecord>(fixture), IClassFixture +{ + [Fact] + public override Task String_array() => Test("StringArray", ["foo", "bar"], ["foo", "baz"], isFilterable: false); + + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + public override Type[] UnsupportedDefaultTypes { get; } = + [ typeof(byte), typeof(short), typeof(decimal), typeof(Guid), typeof(DateTime), typeof(DateTimeOffset), +#if NET + typeof(DateOnly), typeof(TimeOnly), +#endif + ]; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusEmbeddingTypeTests.cs b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusEmbeddingTypeTests.cs new file mode 100644 index 0000000..0e4de5c --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusEmbeddingTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope + +namespace Milvus.ConformanceTests.TypeTests; + +public class MilvusEmbeddingTypeTests(MilvusEmbeddingTypeTests.Fixture fixture) + : EmbeddingTypeTests(fixture), IClassFixture +{ + public new class Fixture : EmbeddingTypeTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} diff --git a/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusKeyTypeTests.cs b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusKeyTypeTests.cs new file mode 100644 index 0000000..1b4c0a5 --- /dev/null +++ b/MEVD/test/Milvus.ConformanceTests/TypeTests/MilvusKeyTypeTests.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Milvus.ConformanceTests.Support; +using VectorData.ConformanceTests.TypeTests; +using VectorData.ConformanceTests.Support; +using Xunit; +namespace Milvus.ConformanceTests.TypeTests; +public class MilvusKeyTypeTests(MilvusKeyTypeTests.Fixture fixture) : KeyTypeTests(fixture), IClassFixture +{ + [Fact] public Task Long() => Test(42L); + [Fact] public Task String() => Test("one", "two"); + [Fact] public Task String_with_non_BMP_characters() => Test("emoji \U0001F600", "emoji \U0001F680"); + [Fact] public Task String_with_escape_characters() => Test("a\"b\\c\n", "d\te\rf"); + public new class Fixture : KeyTypeTests.Fixture + { + public override TestStore TestStore => MilvusTestStore.Instance; + } +} + diff --git a/MEVD/test/Milvus.UnitTests/.editorconfig b/MEVD/test/Milvus.UnitTests/.editorconfig new file mode 100644 index 0000000..394eef6 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/.editorconfig @@ -0,0 +1,6 @@ +# Suppressing errors for Test projects under dotnet folder +[*.cs] +dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task +dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave +dotnet_diagnostic.CS1591.severity = none # Missing XML comment for publicly visible type or member +dotnet_diagnostic.IDE1006.severity = warning # Naming rule violations diff --git a/MEVD/test/Milvus.UnitTests/Milvus.UnitTests.csproj b/MEVD/test/Milvus.UnitTests/Milvus.UnitTests.csproj new file mode 100644 index 0000000..d1f4f6e --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/Milvus.UnitTests.csproj @@ -0,0 +1,36 @@ + + + + CommunityToolkit.VectorData.Milvus.UnitTests + CommunityToolkit.VectorData.Milvus.UnitTests + net10.0 + true + enable + disable + false + $(NoWarn);CA2007,CA1806,CA1869,CA1861,IDE0300,VSTHRD111,SKEXP0001,SKEXP0010,SKEXP0050 + $(NoWarn);MEVD9001 + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + diff --git a/MEVD/test/Milvus.UnitTests/MilvusArrayFilterTests.cs b/MEVD/test/Milvus.UnitTests/MilvusArrayFilterTests.cs new file mode 100644 index 0000000..b4ab429 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusArrayFilterTests.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusArrayFilterTests +{ + private static string Translate(Expression> filter) + => new MilvusFilterTranslator().Translate(filter, + new MilvusModelBuilder().Build(typeof(Record), typeof(string), null, null)); + + [Fact] + public void ArrayContainsGuardsNullFieldsBeforeNegation() + { + Assert.Equal("(tags is not null and array_contains(tags, \"x\"))", Translate(r => r.Tags!.Contains("x"))); + Assert.Equal("(tags is null or not array_contains(tags, \"x\"))", Translate(r => !r.Tags!.Contains("x"))); + Assert.Equal("(ListTags is not null and array_contains(ListTags, \"x\"))", Translate(r => r.ListTags!.Contains("x"))); + } + + [Fact] + public void ArrayAnyGuardsNullFieldsAndDiscardsNullCandidates() + { + var candidates = new[] { "x", null }; + const string expected = "(tags is not null and array_contains_any(tags, [\"x\"]))"; + Assert.Equal(expected, Translate(r => r.Tags!.Any(t => new[] { "x", null }.Contains(t)))); + Assert.Equal(expected, Translate(r => r.Tags!.Any(t => candidates.Contains(t)))); + Assert.Equal("(tags is null or not array_contains_any(tags, [\"x\"]))", Translate(r => !r.Tags!.Any(t => candidates.Contains(t)))); + Assert.Equal("(ListTags is not null and array_contains_any(ListTags, [\"x\"]))", Translate(r => r.ListTags!.Any(t => candidates.Contains(t)))); + } + + [Fact] + public void EmptyAndAllNullCandidatesMatchNothing() + { + var empty = Array.Empty(); + var nulls = new string?[] { null, null }; + Assert.Equal("(Id is null)", Translate(r => r.Tags!.Any(t => empty.Contains(t)))); + Assert.Equal("(Id is null)", Translate(r => r.Tags!.Any(t => nulls.Contains(t)))); + Assert.Equal("(Id is not null)", Translate(r => !r.Tags!.Any(t => nulls.Contains(t)))); + Assert.Equal("(Id is null)", Translate(r => r.Tags!.Contains(null!))); + } + + [Fact] + public void CompoundNegationPreservesNullArrayMembership() + { + Assert.Equal("((tags is null or not array_contains(tags, \"x\")) or (ListTags is null or not array_contains(ListTags, \"y\")))", + Translate(r => !(r.Tags!.Contains("x") && r.ListTags!.Contains("y")))); + Assert.Equal("((tags is null or not array_contains(tags, \"x\")) and (ListTags is null or not array_contains(ListTags, \"y\")))", + Translate(r => !(r.Tags!.Contains("x") || r.ListTags!.Contains("y")))); + Assert.Equal("(tags is not null and array_contains(tags, \"x\"))", Translate(r => !!r.Tags!.Contains("x"))); + } + + public sealed class Record + { + [VectorStoreKey] public string Id { get; set; } = string.Empty; + [VectorStoreData(StorageName = "tags")] public string[]? Tags { get; set; } + [VectorStoreData] public List? ListTags { get; set; } + [VectorStoreVector(3)] public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusArrayTests.cs b/MEVD/test/Milvus.UnitTests/MilvusArrayTests.cs new file mode 100644 index 0000000..c60942a --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusArrayTests.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; +namespace CommunityToolkit.VectorData.Milvus.UnitTests; +public sealed class MilvusArrayTests +{ + [Theory] + [InlineData(typeof(string))] + [InlineData(typeof(bool))] + [InlineData(typeof(int))] + [InlineData(typeof(long))] + [InlineData(typeof(float))] + [InlineData(typeof(double))] + public void SchemaSupportsNativeArrayElements(Type element) + { + var definition = new VectorStoreCollectionDefinition { Properties = + [new VectorStoreKeyProperty("Id", typeof(string)), new VectorStoreDataProperty("Values", element.MakeArrayType()), + new VectorStoreVectorProperty("Vector", typeof(float[]), 3)] }; + var model = new MilvusModelBuilder().BuildDynamic(definition, null); + var field = Assert.Single(MilvusCollectionCreateMapping.MapCollectionDefinition(model), f => f.Name == "Values"); + Assert.Equal(MilvusDataType.Array, field.DataType); + Assert.Equal(MilvusFieldMapping.GetDataType(element), field.ElementDataType); + Assert.True(field.Nullable); + Assert.Equal(4096, field.MaxCapacity); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void MapperReportsRecordPropertyAndIndexForNullArrayElements(bool list) + { + var model = new MilvusModelBuilder().BuildDynamic(new() { Properties = + [new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("DisplayTags", list ? typeof(List) : typeof(string[])) { StorageName = "stored_tags" }, + new VectorStoreVectorProperty("Vector", typeof(float[]), 3)] }, null); + var mapper = new MilvusMapper>(model); + var record = new Dictionary + { + ["Id"] = "record-42", + ["DisplayTags"] = list ? new List { "a", null!, "b" } : new[] { "a", null!, "b" }, + ["Vector"] = new float[] { 1, 0, 0 }, + }; + var exception = Assert.Throws(() => mapper.MapFromDataToStorageModel(record, 3, null)); + Assert.Equal("record", exception.ParamName); + Assert.Contains("record-42", exception.Message); + Assert.Contains("batch index 3", exception.Message); + Assert.Contains("index 1", exception.Message); + Assert.Contains("DisplayTags", exception.Message); + Assert.Contains("stored_tags", exception.Message); + } + + [Fact] + public void MapperAcceptsNullAndEmptyArrayFields() + { + var model = new MilvusModelBuilder().BuildDynamic(new() { Properties = + [new VectorStoreKeyProperty("Id", typeof(string)), new VectorStoreDataProperty("Tags", typeof(string[])), + new VectorStoreVectorProperty("Vector", typeof(float[]), 3)] }, null); + var mapper = new MilvusMapper>(model); + var record = new Dictionary { ["Id"] = "one", ["Tags"] = null, ["Vector"] = new float[] { 1, 0, 0 } }; + Assert.Null(mapper.MapFromDataToStorageModel(record, 0, null)["Tags"]); + record["Tags"] = Array.Empty(); + Assert.Empty(Assert.IsType(mapper.MapFromDataToStorageModel(record, 0, null)["Tags"])); + } + + [Fact] + public void MapperPreservesArraysListsNullsAndEmptyArrays() + { + var model = new MilvusModelBuilder().BuildDynamic(new() { Properties = + [new VectorStoreKeyProperty("Id", typeof(string)), new VectorStoreDataProperty("Array", typeof(string[])), + new VectorStoreDataProperty("List", typeof(List)), new VectorStoreVectorProperty("Vector", typeof(float[]), 3)] }, null); + var mapper = new MilvusMapper>(model); + var mapped = mapper.MapFromStorageToDataModel(new Dictionary + { ["Id"] = "one", ["Array"] = new List { "a", "b" }, ["List"] = new int[] { 1, 2 } }, false); + Assert.Equal(new[] { "a", "b" }, Assert.IsType(mapped["Array"])); + Assert.Equal(new[] { 1, 2 }, Assert.IsType>(mapped["List"])); + mapped = mapper.MapFromStorageToDataModel(new Dictionary + { ["Id"] = "one", ["Array"] = Array.Empty(), ["List"] = null }, false); + Assert.Empty(Assert.IsType(mapped["Array"])); + Assert.Null(mapped["List"]); + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusClientLifetimeTests.cs b/MEVD/test/Milvus.UnitTests/MilvusClientLifetimeTests.cs new file mode 100644 index 0000000..b5a2d4a --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusClientLifetimeTests.cs @@ -0,0 +1,161 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Grpc.Core; +using Grpc.Net.Client; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusClientLifetimeTests +{ + [Fact] + public async Task CollectionsPreserveCallerOwnedGrpcChannel() + { + using var handler = new MilvusHandler(); + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + var store = new MilvusVectorStore(client, ownsClient: true); + using var collection = store.GetCollection("records"); + store.Dispose(); + store.Dispose(); + Assert.True(await collection.CollectionExistsAsync()); + Assert.False(handler.Disposed); + collection.Dispose(); + Assert.False(handler.Disposed); + channel.Dispose(); + Assert.True(handler.Disposed); + } + + [Fact] + public async Task BorrowedClientSurvivesDisposalOfStoreAndCollection() + { + using var handler = new MilvusHandler(); + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + using var store = new MilvusVectorStore(client, ownsClient: false); + using var collection = store.GetCollection("records"); + collection.Dispose(); + store.Dispose(); + Assert.True(await client.HasCollectionAsync("records")); + Assert.False(handler.Disposed); + } + + [Fact] + public async Task WrapsGrpcFailuresWithCollectionMetadata() + { + using var handler = new MilvusHandler { GrpcStatus = "14" }; + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + var exception = await Assert.ThrowsAsync(() => collection.CollectionExistsAsync()); + Assert.IsType(exception.InnerException); + Assert.Equal("milvus", exception.VectorStoreSystemName); + Assert.Equal("records", exception.CollectionName); + Assert.Equal("CollectionExists", exception.OperationName); + } + + [Fact] + public async Task WrapsMilvusFailuresWithOperationMetadata() + { + var metadata = new VectorStoreCollectionMetadata { CollectionName = "records", VectorStoreSystemName = "milvus" }; + var error = new MilvusException("server error"); + var exception = await Assert.ThrowsAsync(() => MilvusErrorHandler.RunAsync(metadata, "Upsert", () => Task.FromException(error))); + Assert.Same(error, exception.InnerException); + Assert.Equal("Upsert", exception.OperationName); + } + + [Fact] + public async Task OwnedClientIsDisposedWhenTheCollectionCannotBeCreated() + { + var unsupportedKey = new MilvusClient("localhost", port: 1); + var invalidModel = new MilvusClient("localhost", port: 1); + var noDefinition = new MilvusClient("localhost", port: 1); + + Assert.Throws(() => new MilvusCollection(unsupportedKey, "records", ownsClient: true)); + Assert.Throws(() => new MilvusCollection(invalidModel, "records", ownsClient: true)); + Assert.Throws(() => new MilvusDynamicCollection(noDefinition, "records", ownsClient: true, new())); + + Assert.IsType(await CallAsync(unsupportedKey)); + Assert.IsType(await CallAsync(invalidModel)); + Assert.IsType(await CallAsync(noDefinition)); + } + + [Fact] + public async Task BorrowedClientSurvivesACollectionThatCannotBeCreated() + { + using var client = new MilvusClient("localhost", port: 1); + + Assert.Throws(() => new MilvusCollection(client, "records", ownsClient: false)); + + Assert.IsNotType(await CallAsync(client)); + } + + [Fact] + public async Task StoreKeepsItsClientWhenOneOfItsCollectionsCannotBeCreated() + { + var client = new MilvusClient("localhost", port: 1); + var store = new MilvusVectorStore(client, ownsClient: true); + + Assert.Throws(() => store.GetCollection("records")); + Assert.IsNotType(await CallAsync(client)); + + store.Dispose(); + Assert.IsType(await CallAsync(client)); + } + + // Calls the client with a canceled token: a disposed client throws ObjectDisposedException, another the cancellation, + // without contacting a server. + private static async Task CallAsync(MilvusClient client) + { + using var canceled = new CancellationTokenSource(); + canceled.Cancel(); + return await Record.ExceptionAsync(() => client.HasCollectionAsync("records", cancellationToken: canceled.Token)); + } + + public sealed class InvalidRecord + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreData(IsFullTextIndexed = true)] + public int Count { get; set; } + + [VectorStoreVector(3)] + public ReadOnlyMemory Vector { get; set; } + } + + private sealed class MilvusHandler : HttpMessageHandler + { + public bool Disposed { get; private set; } + public string GrpcStatus { get; init; } = "0"; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + // gRPC frame containing BoolResponse { status: {}, value: true }. + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Version = new Version(2, 0), + Content = new ByteArrayContent(new byte[] { 0, 0, 0, 0, 4, 10, 0, 16, 1 }), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); + response.TrailingHeaders.Add("grpc-status", GrpcStatus); + return Task.FromResult(response); + } + + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusCollectionTests.cs b/MEVD/test/Milvus.UnitTests/MilvusCollectionTests.cs new file mode 100644 index 0000000..8dd311d --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusCollectionTests.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusCollectionTests +{ + [Fact] + public async Task EmptyBatchesDoNotContactServer() + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + await collection.UpsertAsync(Array.Empty()); + await collection.DeleteAsync(Array.Empty()); + Assert.Empty(await collection.GetAsync(Array.Empty()).ToListAsync()); + } + + [Fact] + public void StoreExposesUnderlyingClientAndMetadata() + { + using var client = new MilvusClient("localhost"); + using var store = new MilvusVectorStore(client, ownsClient: false); + Assert.Same(client, store.GetService(typeof(MilvusClient))); + Assert.Equal("milvus", Assert.IsType(store.GetService(typeof(VectorStoreMetadata))).VectorStoreSystemName); + Assert.Null(store.GetService(typeof(MilvusClient), "key")); + using var collection = store.GetCollection("records"); + Assert.Same(client, collection.GetService(typeof(MilvusClient))); + Assert.Null(collection.GetService(typeof(IKeywordHybridSearchable))); + } + + [Fact] + public void DependencyInjectionResolvesKeyedAndUnkeyedCollections() + { + using var client = new MilvusClient("localhost"); + var services = new ServiceCollection().AddSingleton(client) + .AddMilvusVectorStore() + .AddKeyedMilvusVectorStore("store") + .AddMilvusCollection("records") + .AddKeyedMilvusCollection("collection", "other_records"); + using var provider = services.BuildServiceProvider(); + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + Assert.Same(provider.GetRequiredKeyedService("store"), provider.GetRequiredKeyedService("store")); + Assert.Equal("records", provider.GetRequiredService>().Name); + Assert.Equal("other_records", provider.GetRequiredKeyedService>("collection").Name); + } + + [Fact] + public async Task RejectsInvalidSearchBeforeContactingServer() + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + await Assert.ThrowsAsync(async () => await collection.SearchAsync(new float[] { 1, 2, 3 }, 0).ToListAsync()); + await Assert.ThrowsAsync(async () => await collection.SearchAsync(new float[] { 1 }, 1).ToListAsync()); + } + + [Fact] + public void DynamicCollectionRequiresDefinition() + { + using var client = new MilvusClient("localhost"); + Assert.Throws(() => new MilvusDynamicCollection(client, "records", ownsClient: false, new())); + Assert.Throws(() => new MilvusCollection>(client, "records", ownsClient: false)); + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusDatabaseTests.cs b/MEVD/test/Milvus.UnitTests/MilvusDatabaseTests.cs new file mode 100644 index 0000000..c1290f0 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusDatabaseTests.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System; +using CommunityToolkit.VectorData.UnitTests; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; +namespace CommunityToolkit.VectorData.Milvus.UnitTests; +public sealed class MilvusDatabaseTests +{ + [Fact] + public void OptionsHaveNoDatabase() + { + // The database is a setting of the Milvus client: the options of the store and of the collections do not carry it. + Assert.Null(typeof(MilvusVectorStoreOptions).GetProperty("DatabaseName")); + Assert.Null(typeof(MilvusCollectionOptions).GetProperty("DatabaseName")); + } + + [Fact] + public void SuppliedClientLeavesTheDatabaseOutOfTheMetadata() + { + // Milvus.Client does not expose the database it is configured with. + using var client = new MilvusClient("localhost", database: "catalog"); + using var store = new MilvusVectorStore(client, ownsClient: false); + Assert.Null(Assert.IsType(store.GetService(typeof(VectorStoreMetadata))).VectorStoreName); + using var typed = store.GetCollection("records"); + Assert.Null(Assert.IsType(typed.GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + using var dynamic = store.GetDynamicCollection("records", new() { Properties = + [new VectorStoreKeyProperty("Id", typeof(string)), new VectorStoreVectorProperty("Vector", typeof(float[]), 3)] }); + Assert.Null(Assert.IsType(dynamic.GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + Assert.Null(Assert.IsType(collection.GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + } + + [Fact] + public void HostRegistrationsSupportDatabaseAndBothAuthenticationModes() + { + var services = new ServiceCollection() + .AddMilvusVectorStore("localhost", database: "catalog", username: "root", password: "password") + .AddKeyedMilvusVectorStore("key", "localhost", apiKey: "token", database: "catalog") + .AddMilvusCollection("records", "localhost", database: "catalog", username: "root", password: "password") + .AddKeyedMilvusCollection("key", "records", "localhost", apiKey: "token", database: "catalog"); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + Assert.Equal("catalog", Assert.IsType(store.GetService(typeof(VectorStoreMetadata))).VectorStoreName); + Assert.Equal("catalog", Assert.IsType(store.GetCollection("records").GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + Assert.Equal("catalog", Assert.IsType(provider.GetRequiredKeyedService("key").GetService(typeof(VectorStoreMetadata))).VectorStoreName); + Assert.Equal("catalog", Assert.IsType(provider.GetRequiredService>().GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + Assert.Equal("catalog", Assert.IsType(provider.GetRequiredKeyedService>("key").GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + } + + [Fact] + public void HostRegistrationsWithoutDatabaseReportTheDefaultDatabase() + { + var services = new ServiceCollection() + .AddMilvusVectorStore("localhost", options: new() { EmbeddingGenerator = new FakeEmbeddingGenerator() }) + .AddMilvusCollection("records", "localhost"); + using var provider = services.BuildServiceProvider(); + var store = provider.GetRequiredService(); + Assert.Equal("default", Assert.IsType(store.GetService(typeof(VectorStoreMetadata))).VectorStoreName); + Assert.Equal("default", Assert.IsType(provider.GetRequiredService>().GetService(typeof(VectorStoreCollectionMetadata))).VectorStoreName); + } + + [Fact] + public void RejectsIncompleteOrConflictingCredentials() + { + var services = new ServiceCollection(); + Assert.Throws(() => services.AddMilvusVectorStore("localhost", username: "root")); + Assert.Throws(() => services.AddKeyedMilvusVectorStore("key", "localhost", password: "password")); + Assert.Throws(() => services.AddMilvusCollection("records", "localhost", apiKey: "token", username: "root", password: "password")); + Assert.Throws(() => services.AddKeyedMilvusCollection("key", "records", "localhost", username: "root")); + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusEnsureCollectionTests.cs b/MEVD/test/Milvus.UnitTests/MilvusEnsureCollectionTests.cs new file mode 100644 index 0000000..c0f3bfb --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusEnsureCollectionTests.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Grpc.Net.Client; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusEnsureCollectionTests +{ + [Fact] + public async Task CreatesTheIndexesWithTheCollection() + { + using var handler = new RecordingHandler { CollectionExists = false }; + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + await collection.EnsureCollectionExistsAsync(); + + Assert.Contains("CreateCollection", handler.Methods); + // The vector index and the BM25 index of the full-text property. + Assert.Equal(2, handler.Methods.Count(m => m == "CreateIndex")); + Assert.Contains("LoadCollection", handler.Methods); + } + + [Fact] + public async Task LeavesTheIndexesOfAnExistingCollectionAsTheyAre() + { + using var handler = new RecordingHandler { CollectionExists = true }; + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + await collection.EnsureCollectionExistsAsync(); + + Assert.DoesNotContain("CreateCollection", handler.Methods); + Assert.DoesNotContain("CreateIndex", handler.Methods); + Assert.Contains("LoadCollection", handler.Methods); + } + + [Fact(Timeout = 30000)] + public async Task ReportsALoadThatDoesNotEndAsAVectorStoreException() + { + // The server reports the load at 1% at each poll. + using var handler = new RecordingHandler { CollectionExists = true, LoadingProgress = 1 }; + using var channel = GrpcChannel.ForAddress("http://localhost", new() { HttpHandler = handler, DisposeHttpClient = true }); + using var client = new MilvusClient(channel); + using var collection = new MilvusCollection(client, "records", ownsClient: false) + { + LoadTimeout = TimeSpan.FromMilliseconds(100), + }; + + var exception = await Assert.ThrowsAsync(() => collection.EnsureCollectionExistsAsync(TestContext.Current.CancellationToken)); + + Assert.IsType(exception.InnerException); + Assert.Equal("EnsureCollectionExists", exception.OperationName); + Assert.Equal("records", exception.CollectionName); + } + + [Fact] + public void WaitsForTheLoadLongerThanMilvusDoes() + { + // Milvus 2.6.25 sets queryCoord.loadTimeoutSeconds to 600 in its default configuration. + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + Assert.Equal(TimeSpan.FromMinutes(11), collection.LoadTimeout); + } + + // Answers each gRPC call of Milvus.Client with an empty status, and records the methods called. + internal sealed class RecordingHandler : HttpMessageHandler + { + private readonly List _methods = []; + + public bool CollectionExists { get; init; } + + public int LoadingProgress { get; init; } = 100; + + public IReadOnlyList Methods + { + get + { + lock (_methods) + { + return _methods.ToList(); + } + } + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var method = request.RequestUri!.AbsolutePath.Split('/').Last(); + lock (_methods) + { + _methods.Add(method); + } + + // A gRPC frame of a message with an empty status as field 1 and a varint as field 2: the value of a BoolResponse, + // the progress of a GetLoadingProgressResponse, and a field the other responses ignore. + var value = method switch + { + "HasCollection" => CollectionExists ? 1 : 0, + "GetLoadingProgress" => LoadingProgress, + _ => 1, + }; + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Version = new Version(2, 0), + Content = new ByteArrayContent([0, 0, 0, 0, 4, 10, 0, 16, (byte)value]), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); + response.TrailingHeaders.Add("grpc-status", "0"); + return Task.FromResult(response); + } + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusFilterTranslatorTests.cs b/MEVD/test/Milvus.UnitTests/MilvusFilterTranslatorTests.cs new file mode 100644 index 0000000..b65a453 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusFilterTranslatorTests.cs @@ -0,0 +1,146 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Globalization; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusFilterTranslatorTests +{ + private static string Translate(Expression> expression) + => new MilvusFilterTranslator().Translate(expression, + new MilvusModelBuilder().Build(typeof(MilvusTestRecord), typeof(Guid), null, null)); + + [Fact] + public void UsesStorageNamesAndEscapesStrings() + { + var text = "\" or true or label == \""; + Assert.Equal("(label is not null and (label == " + MilvusFieldMapping.Literal(text) + "))", Translate(r => r.Name == text)); + } + + [Fact] + public void TranslatesLogicalAndNullableComparisons() + => Assert.Equal("((Count is not null and (Count >= 5)) and (not (Enabled == true)))", Translate(r => r.Count >= 5 && !r.Enabled)); + + [Fact] + public void TranslatesNullChecks() + { + Assert.Equal("(Count is null)", Translate(r => r.Count == null)); + Assert.Equal("(label is not null)", Translate(r => r.Name != null)); + } + + [Fact] + public void TranslatesKeyAndCapturedContains() + { + var keys = new[] { Guid.NewGuid(), Guid.NewGuid() }; + var filter = Translate(r => keys.Contains(r.Key)); + Assert.Equal("(record_id in [" + string.Join(", ", keys.Select(k => MilvusFieldMapping.Literal(k))) + "])", filter); + } + + [Fact] + public void EmptyContainsMatchesNothing() + { + var keys = Array.Empty(); + Assert.Equal("(record_id is null)", Translate(r => keys.Contains(r.Key))); + } + +#pragma warning disable CS0464 // Deliberately verify lifted comparisons with literal null. + [Fact] + public void RelationalComparisonsWithNullMatchNothing() + { + int? value = null; + Assert.Equal("(record_id is null)", Translate(r => r.Count > default(int?))); + Assert.Equal("(record_id is null)", Translate(r => r.Count >= value)); + Assert.Equal("(record_id is null)", Translate(r => value < r.Count)); + Assert.Equal("(not (record_id is null))", Translate(r => !(r.Count <= value))); + } + +#pragma warning restore CS0464 + + [Fact] + public void ConstantPredicatesReferenceTheNonNullablePrimaryKey() + { + Assert.Equal("(record_id is not null)", Translate(r => true)); + Assert.Equal("(record_id is null)", Translate(r => false)); + Assert.Equal("(Enabled == true)", Translate(r => r.Enabled)); + Assert.Equal("(Enabled == true)", Translate(r => r.Enabled == true)); + } + + [Fact] + public void InlineContainsUsesScalarLiterals() + => Assert.Equal("(Count is not null and (Count in [1, 2]))", Translate(r => new int?[] { 1, 2 }.Contains(r.Count))); + + [Fact] + public void NullableInequalityAndNegationIncludeNullRows() + { + Assert.Equal("(Count is null or (Count != 5))", Translate(r => r.Count != 5)); + Assert.Equal("(not (Count is not null and (Count > 5)))", Translate(r => !(r.Count > 5))); + } + + [Fact] + public void RejectsUnsupportedMethods() + => Assert.Throws(() => Translate(r => r.Name!.StartsWith("a"))); + + [Fact] + public void NumericLiteralsAreCultureInvariant() + { + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("it-IT"); + Assert.Equal("1.25", MilvusFieldMapping.Literal(1.25)); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Theory] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + public void RejectsNonFiniteNumbers(double value) + => Assert.Throws(() => MilvusFieldMapping.Literal(value)); + + private static string TranslateDynamic(Expression, bool>> expression) + => new MilvusFilterTranslator().Translate(expression, new MilvusModelBuilder().BuildDynamic(new() + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("A", typeof(int?)), + new VectorStoreDataProperty("B", typeof(int?)), + new VectorStoreDataProperty("I", typeof(int)), + new VectorStoreDataProperty("S", typeof(string)), + new VectorStoreDataProperty("T", typeof(string)), + new VectorStoreVectorProperty("Vector", typeof(float[]), 3), + ], + }, null)); + + [Fact] + public void ComparesTwoNullablePropertiesAsCSharpDoes() + { + // == matches two nulls, != matches one null, and a relational operator is false with a null. + Assert.Equal("((A is null and B is null) or (A is not null and B is not null and (A == B)))", TranslateDynamic(r => (int?)r["A"] == (int?)r["B"])); + Assert.Equal("((A is null and B is not null) or (A is not null and B is null) or (A is not null and B is not null and (A != B)))", + TranslateDynamic(r => (int?)r["A"] != (int?)r["B"])); + Assert.Equal("(A is not null and B is not null and (A < B))", TranslateDynamic(r => (int?)r["A"] < (int?)r["B"])); + Assert.Equal("(not (A is not null and B is not null and (A >= B)))", TranslateDynamic(r => !((int?)r["A"] >= (int?)r["B"]))); + Assert.Equal("((S is null and T is null) or (S is not null and T is not null and (S == T)))", TranslateDynamic(r => (string?)r["S"] == (string?)r["T"])); + Assert.Equal("((label is null and label is null) or (label is not null and label is not null and (label == label)))", Translate(r => r.Name == r.Name)); + } + + [Fact] + public void ComparesANullableAndANonNullablePropertyAsCSharpDoes() + { + Assert.Equal("(A is not null and (A == I))", TranslateDynamic(r => (int?)r["A"] == (int)r["I"]!)); + Assert.Equal("(A is null or (I != A))", TranslateDynamic(r => (int)r["I"]! != (int?)r["A"])); + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusFullTextTests.cs b/MEVD/test/Milvus.UnitTests/MilvusFullTextTests.cs new file mode 100644 index 0000000..bfdab94 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusFullTextTests.cs @@ -0,0 +1,192 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using CommunityToolkit.VectorData.UnitTests; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusFullTextTests +{ + private static CollectionModel Build(params VectorStoreProperty[] properties) + => new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(string)), .. properties, new VectorStoreVectorProperty("Vector", typeof(float[]), 3)], + }, null); + + [Fact] + public void FullTextPropertyGetsAnAnalyzerAHiddenSparseFieldAndABm25Function() + { + var schema = MilvusCollectionCreateMapping.MapCollectionSchema(Build( + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true, StorageName = "body" }, + new VectorStoreDataProperty("Title", typeof(string)))); + + var text = Assert.Single(schema.Fields, f => f.Name == "body"); + Assert.Equal(MilvusDataType.VarChar, text.DataType); + Assert.True(text.EnableAnalyzer); + Assert.True(text.Nullable); + Assert.False(Assert.Single(schema.Fields, f => f.Name == "Title").EnableAnalyzer); + Assert.Equal(MilvusDataType.SparseFloatVector, Assert.Single(schema.Fields, f => f.Name == "body_bm25").DataType); + var function = Assert.Single(schema.Functions); + Assert.Equal(FunctionType.Bm25, function.Type); + Assert.Equal("body_bm25", function.Name); + Assert.Equal(["body"], function.InputFieldNames); + Assert.Equal(["body_bm25"], function.OutputFieldNames); + } + + [Fact] + public void EachFullTextPropertyGetsItsOwnBm25Field() + { + var schema = MilvusCollectionCreateMapping.MapCollectionSchema(Build( + new VectorStoreDataProperty("Text1", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreDataProperty("Text2", typeof(string)) { IsFullTextIndexed = true })); + + Assert.Equal(["Text1_bm25", "Text2_bm25"], schema.Fields.Where(f => f.DataType == MilvusDataType.SparseFloatVector).Select(f => f.Name)); + Assert.Equal(["Text1", "Text2"], schema.Functions.SelectMany(f => f.InputFieldNames)); + } + + [Fact] + public void ModelWithoutFullTextHasNoBm25FieldOrFunction() + { + var schema = MilvusCollectionCreateMapping.MapCollectionSchema(Build(new VectorStoreDataProperty("Text", typeof(string)) { IsIndexed = true })); + + Assert.DoesNotContain(schema.Fields, f => f.DataType == MilvusDataType.SparseFloatVector || f.EnableAnalyzer); + Assert.Empty(schema.Functions); + } + + [Theory] + [InlineData(typeof(int))] + [InlineData(typeof(bool?))] + [InlineData(typeof(string[]))] + [InlineData(typeof(List))] + public void RejectsFullTextOnPropertiesThatAreNotStrings(Type type) + { + var exception = Assert.Throws(() => Build(new VectorStoreDataProperty("Text", type) { IsFullTextIndexed = true })); + Assert.Contains("'Text'", exception.Message); + Assert.Contains("string", exception.Message); + } + + [Theory] + [InlineData("data")] + [InlineData("vector")] + [InlineData("fulltext")] + public void RejectsAPropertyStoredUnderTheNameOfAHiddenBm25Field(string kind) + { + VectorStoreProperty conflict = kind switch + { + "data" => new VectorStoreDataProperty("Other", typeof(string)) { StorageName = "Text_bm25" }, + "vector" => new VectorStoreVectorProperty("Other", typeof(float[]), 3) { StorageName = "Text_bm25" }, + _ => new VectorStoreDataProperty("Other", typeof(string)) { StorageName = "Text_bm25", IsFullTextIndexed = true }, + }; + + var exception = Assert.Throws(() => Build(new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, conflict)); + Assert.Contains("'Other'", exception.Message); + Assert.Contains("'Text_bm25'", exception.Message); + Assert.Contains("'Text'", exception.Message); + } + + [Fact] + public void RejectsAKeyStoredUnderTheNameOfAHiddenBm25Field() + { + var exception = Assert.Throws(() => new MilvusModelBuilder().BuildDynamic(new() + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)) { StorageName = "Text_bm25" }, + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreVectorProperty("Vector", typeof(float[]), 3), + ], + }, null)); + Assert.Contains("'Text_bm25'", exception.Message); + } + + [Fact] + public void RejectsFullTextStorageNamesTooLongForTheBm25Field() + { + // Milvus field names allow up to 255 characters, and the BM25 field adds "_bm25" to the storage name. + Build(new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true, StorageName = new string('t', 250) }); + var exception = Assert.Throws(() => Build(new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true, StorageName = new string('t', 251) })); + Assert.Contains("255", exception.Message); + } + + [Fact] + public void SupportsTheDefinitionOfTheChatHistoryMemoryProviderOfAgentFramework() + { + // The definition ChatHistoryMemoryProvider of Microsoft.Agents.AI 1.23.0 creates, as a dynamic collection. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(Guid)), + new VectorStoreDataProperty("Role", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("MessageId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("AuthorName", typeof(string)), + new VectorStoreDataProperty("ApplicationId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("AgentId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("UserId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("SessionId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("Content", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreDataProperty("CreatedAt", typeof(string)) { IsIndexed = true }, + new VectorStoreVectorProperty("ContentEmbedding", typeof(string), 3), + ], + }; + using var client = new MilvusClient("localhost", port: 1); + using var store = new MilvusVectorStore(client, ownsClient: false, new() { EmbeddingGenerator = new FakeEmbeddingGenerator() }); + + using var collection = store.GetDynamicCollection("chat_history", definition); + + var schema = MilvusCollectionCreateMapping.MapCollectionSchema(new MilvusModelBuilder().BuildDynamic(definition, new FakeEmbeddingGenerator())); + Assert.True(Assert.Single(schema.Fields, f => f.Name == "Content").EnableAnalyzer); + Assert.Equal("Content", Assert.Single(Assert.Single(schema.Functions).InputFieldNames)); + } + + [Fact] + public void SupportsTheDefinitionOfTheTextSearchStoreOfSemanticKernel() + { + // TextSearchStore of Microsoft.SemanticKernel.Core 1.80.1 creates a typed collection with this definition. + using var client = new MilvusClient("localhost", port: 1); + using var store = new MilvusVectorStore(client, ownsClient: false, new() { EmbeddingGenerator = new FakeEmbeddingGenerator() }); + + using var collection = store.GetCollection("docs", TextSearchStoreDocument.Definition(3)); + + // TextSearchStore searches with hybrid search when the collection offers it. + Assert.Same(collection, collection.GetService(typeof(IKeywordHybridSearchable))); + } + + internal sealed class TextSearchStoreDocument + { + public string Key { get; set; } = string.Empty; + + public List Namespaces { get; set; } = []; + + public string? Text { get; set; } + + public string? SourceId { get; set; } + + public string? SourceName { get; set; } + + public string? SourceLink { get; set; } + + public string? TextEmbedding { get; set; } + + public static VectorStoreCollectionDefinition Definition(int dimensions) => new() + { + Properties = + [ + new VectorStoreKeyProperty("Key", typeof(string)), + new VectorStoreDataProperty("Namespaces", typeof(List)) { IsIndexed = true }, + new VectorStoreDataProperty("SourceId", typeof(string)) { IsIndexed = true }, + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreDataProperty("SourceName", typeof(string)), + new VectorStoreDataProperty("SourceLink", typeof(string)), + new VectorStoreVectorProperty("TextEmbedding", typeof(string), dimensions), + ], + }; + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusHybridSearchTests.cs b/MEVD/test/Milvus.UnitTests/MilvusHybridSearchTests.cs new file mode 100644 index 0000000..94bf9d0 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusHybridSearchTests.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using CommunityToolkit.VectorData.UnitTests; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.VectorData; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusHybridSearchTests +{ + private static readonly VectorStoreCollectionDefinition s_definition = new() + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true }, + new VectorStoreVectorProperty("Embedding", typeof(string), 3), + ], + }; + + [Fact] + public void GetServiceReturnsTheCollectionWhenAPropertyIsFullTextIndexed() + { + using var client = new MilvusClient("localhost", port: 1); + using var typed = new MilvusCollection(client, "typed", ownsClient: false); + using var dynamic = new MilvusDynamicCollection(client, "dynamic", ownsClient: false, new() { Definition = s_definition, EmbeddingGenerator = new FakeEmbeddingGenerator() }); + using var withoutText = new MilvusCollection(client, "without_text", ownsClient: false); + + Assert.Same(typed, typed.GetService(typeof(IKeywordHybridSearchable))); + Assert.Same(dynamic, dynamic.GetService(typeof(IKeywordHybridSearchable>))); + Assert.Null(withoutText.GetService(typeof(IKeywordHybridSearchable))); + Assert.Null(typed.GetService(typeof(IKeywordHybridSearchable), "key")); + } + + [Fact] + public void DependencyInjectionResolvesHybridSearch() + { + using var client = new MilvusClient("localhost"); + using var provider = new ServiceCollection().AddSingleton(client) + .AddMilvusCollection("records") + .AddKeyedMilvusCollection("key", "other_records") + .BuildServiceProvider(); + + Assert.Same(provider.GetRequiredService>(), provider.GetRequiredService>()); + Assert.Same(provider.GetRequiredKeyedService>("key"), provider.GetRequiredKeyedService>("key")); + } + + [Fact] + public async Task RejectsInvalidHybridSearchesBeforeContactingTheServer() + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1, 2, 3 }, ["apples"], 0).ToListAsync()); + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1, 2, 3 }, null!, 1).ToListAsync()); + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1 }, ["apples"], 1).ToListAsync()); + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1, 2, 3 }, ["apples"], 1, + new() { AdditionalProperty = r => r.Title }).ToListAsync()); + } + + [Theory] + [InlineData(16384, 1)] + [InlineData(1, 16384)] + [InlineData(int.MaxValue, 1)] + public async Task RejectsTopAndSkipBeyondTheMilvusLimitBeforeContactingTheServer(int top, int skip) + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + var exception = await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1, 2, 3 }, ["apples"], top, + new() { Skip = skip }).ToListAsync()); + Assert.Equal("top", exception.ParamName); + Assert.Contains("16384", exception.Message); + } + + [Fact] + public async Task RejectsIncludeVectorsWithEmbeddingGeneration() + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusDynamicCollection(client, "records", ownsClient: false, new() { Definition = s_definition, EmbeddingGenerator = new FakeEmbeddingGenerator() }); + + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync("apples", ["apples"], 1, new() { IncludeVectors = true }).ToListAsync()); + } + + [Fact] + public async Task RejectsHybridSearchWithoutAFullTextProperty() + { + using var client = new MilvusClient("localhost", port: 1); + using var collection = new MilvusCollection(client, "records", ownsClient: false); + + await Assert.ThrowsAsync(async () => await collection.HybridSearchAsync(new float[] { 1, 2, 3 }, ["apples"], 1).ToListAsync()); + } + + public sealed class TextRecord + { + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + [VectorStoreData(IsFullTextIndexed = true)] + public string? Text { get; set; } + + [VectorStoreData] + public string? Title { get; set; } + + [VectorStoreVector(3)] + public ReadOnlyMemory Vector { get; set; } + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusMappingTests.cs b/MEVD/test/Milvus.UnitTests/MilvusMappingTests.cs new file mode 100644 index 0000000..b0ac371 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusMappingTests.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Milvus.Client; +using Xunit; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusMappingTests +{ + private static CollectionModel Model() => new MilvusModelBuilder().Build(typeof(MilvusTestRecord), typeof(Guid), null, null); + + [Fact] + public void SchemaUsesNativeFieldsAndStorageNames() + { + var fields = MilvusCollectionCreateMapping.MapCollectionDefinition(Model()); + var key = Assert.Single(fields, f => f.IsPrimaryKey); + Assert.Equal("record_id", key.Name); + Assert.Equal(MilvusDataType.VarChar, key.DataType); + Assert.False(key.AutoId); + Assert.True(Assert.Single(fields, f => f.Name == "label").Nullable); + var count = Assert.Single(fields, f => f.Name == "Count"); + Assert.True(count.Nullable); + Assert.Equal(MilvusDataType.Int32, count.DataType); + Assert.Equal(3, Assert.Single(fields, f => f.Name == "Vector").Dimension); + } + + [Fact] + public void LongKeyUsesInt64() + { + var model = new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(long)), new VectorStoreVectorProperty("Vector", typeof(float[]), 3)], + }, null); + Assert.Equal(MilvusDataType.Int64, MilvusCollectionCreateMapping.MapCollectionDefinition(model)[0].DataType); + } + + [Fact] + public void MapperRoundTripsNullsGuidAndVectors() + { + var mapper = new MilvusMapper(Model()); + var record = new MilvusTestRecord { Key = Guid.NewGuid(), Name = null, Count = null, Enabled = true, Vector = new float[] { 1, 2, 3 } }; + var stored = mapper.MapFromDataToStorageModel(record, 0, null); + Assert.Equal(record.Key.ToString("D"), stored["record_id"]); + var result = mapper.MapFromStorageToDataModel(new Dictionary(stored), true); + Assert.Equal(record.Key, result.Key); + Assert.Null(result.Name); + Assert.Null(result.Count); + Assert.True(result.Enabled); + Assert.Equal(record.Vector.ToArray(), result.Vector.ToArray()); + } + + [Fact] + public void MapperExcludesVectorsWhenRequested() + { + var mapper = new MilvusMapper(Model()); + var result = mapper.MapFromStorageToDataModel(new Dictionary + { + ["record_id"] = Guid.NewGuid().ToString("D"), ["Vector"] = new float[] { 1, 2, 3 }, + }, false); + Assert.True(result.Vector.IsEmpty); + } + + [Fact] + public void MapperRejectsWrongDimensions() + { + var mapper = new MilvusMapper(Model()); + Assert.Throws(() => mapper.MapFromDataToStorageModel(new() { Vector = new float[] { 1 } }, 0, null)); + } + + [Theory] + [InlineData(null, SimilarityMetricType.Cosine)] + [InlineData(DistanceFunction.CosineSimilarity, SimilarityMetricType.Cosine)] + [InlineData(DistanceFunction.DotProductSimilarity, SimilarityMetricType.Ip)] + [InlineData(DistanceFunction.EuclideanSquaredDistance, SimilarityMetricType.L2)] + [InlineData(DistanceFunction.EuclideanDistance, SimilarityMetricType.L2)] + public void MapsSupportedDistances(string? distance, SimilarityMetricType expected) + => Assert.Equal(expected, MilvusCollectionCreateMapping.GetMetric(new VectorPropertyModel("Vector", typeof(float[])) { Dimensions = 3, DistanceFunction = distance })); + + [Theory] + [InlineData(DistanceFunction.ManhattanDistance)] + [InlineData(DistanceFunction.CosineDistance)] + public void RejectsUnsupportedDistances(string distance) + => Assert.Throws(() => MilvusCollectionCreateMapping.GetMetric(new VectorPropertyModel("Vector", typeof(float[])) { Dimensions = 3, DistanceFunction = distance })); + + [Theory] + [InlineData(null, IndexType.AutoIndex)] + [InlineData(IndexKind.Flat, IndexType.Flat)] + [InlineData(IndexKind.Hnsw, IndexType.Hnsw)] + public void MapsSupportedIndexes(string? kind, IndexType expected) + => Assert.Equal(expected, MilvusCollectionCreateMapping.GetIndexType(new VectorPropertyModel("Vector", typeof(float[])) { Dimensions = 3, IndexKind = kind })); + + [Theory] + [InlineData(DistanceFunction.CosineSimilarity, .8, .7, true)] + [InlineData(DistanceFunction.CosineSimilarity, .6, .7, false)] + [InlineData(DistanceFunction.EuclideanSquaredDistance, .6, .7, true)] + [InlineData(DistanceFunction.EuclideanSquaredDistance, .8, .7, false)] + [InlineData(DistanceFunction.EuclideanDistance, .6, .7, true)] + [InlineData(DistanceFunction.EuclideanDistance, .8, .7, false)] + public void AppliesMetricThresholdDirection(string distance, double score, double threshold, bool expected) + => Assert.Equal(expected, MilvusCollectionSearchMapping.PassesThreshold(score, threshold, distance)); + + [Theory] + [InlineData(DistanceFunction.EuclideanDistance, 4f, 2d)] + [InlineData(DistanceFunction.EuclideanSquaredDistance, 4f, 4d)] + [InlineData(DistanceFunction.CosineSimilarity, .5f, .5d)] + [InlineData(DistanceFunction.DotProductSimilarity, -2f, -2d)] + [InlineData(null, .5f, .5d)] + public void MapsMilvusScores(string? distance, float score, double expected) + => Assert.Equal(expected, MilvusCollectionSearchMapping.MapScore(score, distance)); + + [Theory] + [InlineData(typeof(ulong))] + [InlineData(typeof(int))] + public void RejectsUnsupportedKeyTypes(Type keyType) + => Assert.Throws(() => new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", keyType), new VectorStoreVectorProperty("Vector", typeof(float[]), 3)], + }, null)); + + [Fact] + public void RejectsAutoGeneratedStringKeysDuringModelConstruction() + { + var exception = Assert.Throws(() => new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(string)) { IsAutoGenerated = true }, + new VectorStoreVectorProperty("Vector", typeof(float[]), 3)], + }, null)); + Assert.Contains("Id", exception.Message); + Assert.Contains("auto-generation", exception.Message); + } + + [Theory] + [InlineData("like")] + [InlineData("AND")] + [InlineData("Array_Contains")] + [InlineData("true")] + [InlineData("bad-name")] + [InlineData("0name")] + [InlineData("a] or true")] + public void RejectsInvalidFieldNames(string name) + => Assert.Throws(() => new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(string)) { StorageName = name }, new VectorStoreVectorProperty("Vector", typeof(float[]), 3)], + }, null)); + + [Fact] + public void SupportsMultipleNamedVectorsAndGeneratedEmbeddings() + { + var model = new MilvusModelBuilder().BuildDynamic(new() + { + Properties = [new VectorStoreKeyProperty("Id", typeof(string)), + new VectorStoreVectorProperty("First", typeof(float[]), 3), new VectorStoreVectorProperty("Second", typeof(Embedding), 3)], + }, null); + var mapper = new MilvusMapper>(model); + var row = mapper.MapFromDataToStorageModel(new() + { + ["Id"] = "one", ["First"] = new float[] { 1, 2, 3 }, ["Second"] = new Embedding(new float[] { 3, 2, 1 }), + }, 0, null); + var result = mapper.MapFromStorageToDataModel(new Dictionary(row), true); + Assert.Equal(new float[] { 1, 2, 3 }, Assert.IsType(result["First"])); + Assert.Equal(new float[] { 3, 2, 1 }, Assert.IsType>(result["Second"]).Vector.ToArray()); + } +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusStringEscapingTests.cs b/MEVD/test/Milvus.UnitTests/MilvusStringEscapingTests.cs new file mode 100644 index 0000000..7b8c8f3 --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusStringEscapingTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; +namespace CommunityToolkit.VectorData.Milvus.UnitTests; +public sealed class MilvusStringEscapingTests +{ + [Fact] + public void StringLiteralsPreserveNonBmpCharactersAndEscapeControlCharacters() + { + Assert.Equal("\"emoji \U0001F600\"", MilvusFieldMapping.Literal("emoji \U0001F600")); + Assert.Equal("\"a\\\"b\\\\c\\n\\r\\t\"", MilvusFieldMapping.Literal("a\"b\\c\n\r\t")); + } + +} diff --git a/MEVD/test/Milvus.UnitTests/MilvusTestRecord.cs b/MEVD/test/Milvus.UnitTests/MilvusTestRecord.cs new file mode 100644 index 0000000..2d30e9b --- /dev/null +++ b/MEVD/test/Milvus.UnitTests/MilvusTestRecord.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.VectorData; + +namespace CommunityToolkit.VectorData.Milvus.UnitTests; + +public sealed class MilvusTestRecord +{ + [VectorStoreKey(StorageName = "record_id")] + public Guid Key { get; set; } + + [VectorStoreData(StorageName = "label")] + public string? Name { get; set; } + + [VectorStoreData] + public int? Count { get; set; } + + [VectorStoreData] + public bool Enabled { get; set; } + + [VectorStoreVector(3, DistanceFunction = DistanceFunction.CosineSimilarity)] + public ReadOnlyMemory Vector { get; set; } +} diff --git a/README.md b/README.md index 38ce531..4578bf8 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The following NuGet packages have been published: | [CommunityToolkit.VectorData.AzureDocumentDB] | ![AzureDocumentDB Version][v-azuredocumentdb] | ![AzureDocumentDB Downloads][d-azuredocumentdb] | | [CommunityToolkit.VectorData.Chroma] | ![Chroma Version][v-chroma] | ![Chroma Downloads][d-chroma] | | [CommunityToolkit.VectorData.InMemory] | ![InMemory Version][v-inmemory] | ![InMemory Downloads][d-inmemory] | +| [CommunityToolkit.VectorData.Milvus] | ![Milvus Version][v-milvus] | ![Milvus Downloads][d-milvus] | | [CommunityToolkit.VectorData.PgVector] | ![PgVector Version][v-pgvector] | ![PgVector Downloads][d-pgvector] | | [CommunityToolkit.VectorData.Qdrant] | ![Qdrant Version][v-qdrant] | ![Qdrant Downloads][d-qdrant] | | [CommunityToolkit.VectorData.Redis] | ![Redis Version][v-redis] | ![Redis Downloads][d-redis] | @@ -57,6 +58,7 @@ The Vector Data connectors in this toolkit were initially part of the [Semantic [CommunityToolkit.VectorData.AzureDocumentDB]: https://www.nuget.org/packages/CommunityToolkit.VectorData.AzureDocumentDB [CommunityToolkit.VectorData.Chroma]: https://www.nuget.org/packages/CommunityToolkit.VectorData.Chroma [CommunityToolkit.VectorData.InMemory]: https://www.nuget.org/packages/CommunityToolkit.VectorData.InMemory +[CommunityToolkit.VectorData.Milvus]: https://www.nuget.org/packages/CommunityToolkit.VectorData.Milvus [CommunityToolkit.VectorData.PgVector]: https://www.nuget.org/packages/CommunityToolkit.VectorData.PgVector [CommunityToolkit.VectorData.Qdrant]: https://www.nuget.org/packages/CommunityToolkit.VectorData.Qdrant [CommunityToolkit.VectorData.Redis]: https://www.nuget.org/packages/CommunityToolkit.VectorData.Redis @@ -70,6 +72,7 @@ The Vector Data connectors in this toolkit were initially part of the [Semantic [v-azuredocumentdb]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.AzureDocumentDB [v-chroma]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.Chroma [v-inmemory]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.InMemory +[v-milvus]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.Milvus [v-pgvector]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.PgVector [v-qdrant]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.Qdrant [v-redis]: https://badgen.net/nuget/v/CommunityToolkit.VectorData.Redis @@ -83,6 +86,7 @@ The Vector Data connectors in this toolkit were initially part of the [Semantic [d-azuredocumentdb]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.AzureDocumentDB [d-chroma]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.Chroma [d-inmemory]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.InMemory +[d-milvus]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.Milvus [d-pgvector]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.PgVector [d-qdrant]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.Qdrant [d-redis]: https://badgen.net/nuget/dt/CommunityToolkit.VectorData.Redis