.Net: Add MEVD TestSuiteImplementationTests (#12574)

roji · web-flow · commit cd34dcd59132 · 2025-07-24T10:52:50.000Z
I noticed that actual test suite implementation is a bit spotty across providers; so this adds a test suite which tests that all other test suites are implemented. We have this in the EF tests as well, also as a way of letting provider maintainers know when a new test suite is introduced which they probably want to implement. Of course, providers can explicitly ignore specific test suites which don't make sense for them. Part of #10194
diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchTestSuiteImplementationTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/AzureAISearchTestSuiteImplementationTests.cs
@@ -0,0 +1,15 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace AzureAISearch.ConformanceTests;
+
+public class AzureAISearchTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+    ];
+}
diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoTestSuiteImplementationTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoTestSuiteImplementationTests.cs
@@ -0,0 +1,21 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.CRUD;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace CosmosMongoDB.ConformanceTests;
+
+public class CosmosMongoTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+        typeof(DynamicDataModelConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>),
+    ];
+}
diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingGenerationTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlEmbeddingGenerationTests.cs
@@ -10,8 +10,8 @@
 
 namespace CosmosNoSql.ConformanceTests;
 
-public class CosmosNoSQLEmbeddingGenerationTests(CosmosNoSQLEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, CosmosNoSQLEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture)
-    : EmbeddingGenerationTests<string>(stringVectorFixture, romOfFloatVectorFixture), IClassFixture<CosmosNoSQLEmbeddingGenerationTests.StringVectorFixture>, IClassFixture<CosmosNoSQLEmbeddingGenerationTests.RomOfFloatVectorFixture>
+public class CosmosNoSqlEmbeddingGenerationTests(CosmosNoSqlEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, CosmosNoSqlEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture)
+    : EmbeddingGenerationTests<string>(stringVectorFixture, romOfFloatVectorFixture), IClassFixture<CosmosNoSqlEmbeddingGenerationTests.StringVectorFixture>, IClassFixture<CosmosNoSqlEmbeddingGenerationTests.RomOfFloatVectorFixture>
 {
     public new class StringVectorFixture : EmbeddingGenerationTests<string>.StringVectorFixture
     {
diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlTestSuiteImplementationTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSqlTestSuiteImplementationTests.cs
@@ -0,0 +1,18 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.CRUD;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace CosmosNoSql.ConformanceTests;
+
+public class CosmosNoSqlTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+        typeof(DynamicDataModelConformanceTests<>),
+        typeof(BatchConformanceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/InMemory.ConformanceTests/InMemoryTestSuiteImplementationTests.cs b/dotnet/test/VectorData/InMemory.ConformanceTests/InMemoryTestSuiteImplementationTests.cs
@@ -0,0 +1,20 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace InMemory.ConformanceTests;
+
+public class InMemoryTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(DependencyInjectionTests<,,,>),
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoTestSuiteImplementationTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/MongoTestSuiteImplementationTests.cs
@@ -0,0 +1,17 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.CRUD;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace MongoDB.ConformanceTests;
+
+public class MongoTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+        typeof(DynamicDataModelConformanceTests<>),
+    ];
+}
diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/PgVectorTestSuiteImplementationTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/PgVectorTestSuiteImplementationTests.cs
@@ -0,0 +1,18 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace PgVector.ConformanceTests;
+
+public class PostgresTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchWithFilterConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeTestSuiteImplementationTests.cs b/dotnet/test/VectorData/Pinecone.ConformanceTests/PineconeTestSuiteImplementationTests.cs
@@ -0,0 +1,18 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace Pinecone.ConformanceTests;
+
+public class PineconeTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchWithFilterConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantTestSuiteImplementationTests.cs b/dotnet/test/VectorData/Qdrant.ConformanceTests/QdrantTestSuiteImplementationTests.cs
@@ -0,0 +1,15 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace Qdrant.ConformanceTests;
+
+public class QdrantTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/Redis.ConformanceTests/RedisTestSuiteImplementationTests.cs b/dotnet/test/VectorData/Redis.ConformanceTests/RedisTestSuiteImplementationTests.cs
@@ -0,0 +1,21 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.CRUD;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace Redis.ConformanceTests;
+
+public class RedisTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(VectorSearchWithFilterConformanceTests<>),
+        typeof(BatchConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerTestSuiteImplementationTests.cs b/dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerTestSuiteImplementationTests.cs
@@ -0,0 +1,18 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace SqlServer.ConformanceTests;
+
+public class SqlServerTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchWithFilterConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteEmbeddingTypeTests.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteEmbeddingTypeTests.cs
@@ -7,7 +7,7 @@
 
 #pragma warning disable CA2000 // Dispose objects before losing scope
 
-namespace SqliteIntegrationTests;
+namespace SqliteVec.ConformanceTests;
 
 public class SqliteEmbeddingTypeTests(SqliteEmbeddingTypeTests.Fixture fixture)
     : EmbeddingTypeTests<string>(fixture), IClassFixture<SqliteEmbeddingTypeTests.Fixture>
diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteTestSuiteImplementationTests.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/SqliteTestSuiteImplementationTests.cs
@@ -0,0 +1,20 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.CRUD;
+using VectorData.ConformanceTests.HybridSearch;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace SqliteVec.ConformanceTests;
+
+public class SqliteTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchDistanceFunctionComplianceTests<>),
+        typeof(DynamicDataModelConformanceTests<>),
+
+        // Hybrid search not supported
+        typeof(KeywordVectorizedHybridSearchComplianceTests<>)
+    ];
+}
diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/TestSuiteImplementationTests.cs b/dotnet/test/VectorData/VectorData.ConformanceTests/TestSuiteImplementationTests.cs
@@ -0,0 +1,83 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using System.Reflection;
+using System.Text.RegularExpressions;
+using Xunit;
+
+namespace VectorData.ConformanceTests;
+
+/// <summary>
+/// A test that ensures that all base test suites are implemented (or explicitly ignored) in provider implementations.
+/// Used to make sure that test coverage is complete.
+/// </summary>
+public abstract class TestSuiteImplementationTests
+{
+    protected virtual ICollection<Type> IgnoredTestBases { get; } = [];
+
+    [Fact]
+    public virtual void All_test_bases_must_be_implemented()
+    {
+        var concreteTests
+            = this.GetType().Assembly.GetTypes()
+                .Where(c => c.BaseType != typeof(object) && !c.IsAbstract && (c.IsPublic || c.IsNestedPublic))
+                .ToList();
+
+        var nonImplementedBases
+            = this.GetBaseTestClasses()
+                .Where(t => !this.IgnoredTestBases.Contains(t) && !concreteTests.Any(c => Implements(c, t)))
+                .Select(t => t.FullName)
+                .ToList();
+
+        Assert.False(
+            nonImplementedBases.Count > 0,
+            "\r\n-- Missing derived classes for --\r\n" + string.Join(Environment.NewLine, nonImplementedBases));
+    }
+
+    // Filter for abstract base types which end with Tests and possibly generic arity (e.g. FooTests`2)
+    protected virtual IEnumerable<Type> GetBaseTestClasses()
+        => typeof(TestSuiteImplementationTests).Assembly.ExportedTypes
+            .Where(t => Regex.IsMatch(t.Name, """Tests(`\d+)?$""") && t.IsAbstract && !t.IsSealed && !t.IsInterface);
+
+    private static bool Implements(Type type, Type interfaceOrBaseType)
+        => (type.IsPublic || type.IsNestedPublic) && interfaceOrBaseType.IsGenericTypeDefinition
+            ? GetGenericTypeImplementations(type, interfaceOrBaseType).Any()
+            : interfaceOrBaseType.IsAssignableFrom(type);
+
+    private static IEnumerable<Type> GetGenericTypeImplementations(Type type, Type interfaceOrBaseType)
+    {
+        var typeInfo = type.GetTypeInfo();
+
+        if (!typeInfo.IsGenericTypeDefinition)
+        {
+            var baseTypes = interfaceOrBaseType.IsInterface
+                ? typeInfo.ImplementedInterfaces
+                : GetBaseTypes(type);
+            foreach (var baseType in baseTypes)
+            {
+                if (baseType.IsGenericType
+                    && baseType.GetGenericTypeDefinition() == interfaceOrBaseType)
+                {
+                    yield return baseType;
+                }
+            }
+
+            if (type.IsGenericType
+                && type.GetGenericTypeDefinition() == interfaceOrBaseType)
+            {
+                yield return type;
+            }
+        }
+    }
+
+    private static IEnumerable<Type> GetBaseTypes(Type type)
+    {
+        var t = type.BaseType;
+
+        while (t != null)
+        {
+            yield return t;
+
+            t = t.BaseType;
+        }
+    }
+}
diff --git a/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateTestSuiteImplementationTests.cs b/dotnet/test/VectorData/Weaviate.ConformanceTests/WeaviateTestSuiteImplementationTests.cs
@@ -0,0 +1,14 @@
+﻿// Copyright (c) Microsoft. All rights reserved.
+
+using VectorData.ConformanceTests;
+using VectorData.ConformanceTests.VectorSearch;
+
+namespace Weaviate.ConformanceTests;
+
+public class WeaviateTestSuiteImplementationTests : TestSuiteImplementationTests
+{
+    protected override ICollection<Type> IgnoredTestBases { get; } =
+    [
+        typeof(VectorSearchWithFilterConformanceTests<>)
+    ];
+}

Original file line number	Diff line number	Diff line change
`@@ -10,8 +10,8 @@`
`10`	`10`
`11`	`11`	`namespace CosmosNoSql.ConformanceTests;`
`12`	`12`
`13`		`-public class CosmosNoSQLEmbeddingGenerationTests(CosmosNoSQLEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, CosmosNoSQLEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture)`
`14`		`- : EmbeddingGenerationTests<string>(stringVectorFixture, romOfFloatVectorFixture), IClassFixture<CosmosNoSQLEmbeddingGenerationTests.StringVectorFixture>, IClassFixture<CosmosNoSQLEmbeddingGenerationTests.RomOfFloatVectorFixture>`
	`13`	`+public class CosmosNoSqlEmbeddingGenerationTests(CosmosNoSqlEmbeddingGenerationTests.StringVectorFixture stringVectorFixture, CosmosNoSqlEmbeddingGenerationTests.RomOfFloatVectorFixture romOfFloatVectorFixture)`
	`14`	`+ : EmbeddingGenerationTests<string>(stringVectorFixture, romOfFloatVectorFixture), IClassFixture<CosmosNoSqlEmbeddingGenerationTests.StringVectorFixture>, IClassFixture<CosmosNoSqlEmbeddingGenerationTests.RomOfFloatVectorFixture>`
`15`	`15`	`{`
`16`	`16`	`public new class StringVectorFixture : EmbeddingGenerationTests<string>.StringVectorFixture`
`17`	`17`	`{`