Give MSTest tests under motus run a TestContext, data rows and MSTest's cleanup order

docs/guides/flaky-tests-and-quarantine.md

@@ -60,7 +60,7 @@ Or supply a list file, which quarantines without touching the source. That suits
 motus run bin/Release/net8.0/MyTests.dll --quarantine quarantine.txt
 ```
 
-One fully qualified test name per line. Blank lines are ignored, and a line whose first non-blank character is `#` is a comment. The whole line is the name, so a trailing `# reason` on the same line becomes part of it and the entry silently matches nothing; put the reason on its own line above.
+One fully qualified test name per line. A data-driven test is listed by the name of one row, such as `MyTests.CheckoutTests.PaysBy (card)`, or by its method, `MyTests.CheckoutTests.PaysBy`, which covers every row. Blank lines are ignored, and a line whose first non-blank character is `#` is a comment. The whole line is the name, so a trailing `# reason` on the same line becomes part of it and the entry silently matches nothing; put the reason on its own line above.
 
 ```
 # Flaky since the payment provider sandbox migration (#412)

docs/guides/testing-frameworks.md

@@ -27,7 +27,7 @@ Inherit from `MotusTestBase` to receive a fully managed `IBrowserContext` and `I
 |---|---|
 | `protected IBrowserContext Context` | The context for the current test. Available after `[TestInitialize]`. |
 | `protected IPage Page` | The page for the current test. Available after `[TestInitialize]`. |
-| `public TestContext TestContext` | Set by the MSTest runner. Used internally to detect test outcome for failure tracing. |
+| `public TestContext TestContext` | Set by MSTest, and by `motus run` and the visual runner. Used internally to detect test outcome for failure tracing. |
 
 ### Browser lifecycle
 

docs/reference/cli.md

@@ -63,6 +63,13 @@ motus run bin/Release/net8.0/MyTests.dll --shard 1/4 --retries 2
 
 The output path for a reporter is part of its value, so it is `--reporter junit:results.xml` rather than a separate flag. Passing `--reporter junit` with no path writes nowhere.
 
+**How tests run.** `motus run` and the visual runner load the test assembly and call its tests by reflection, without going through the MSTest, NUnit or xUnit test adapter. They cover the parts of each framework that suites lean on:
+
+- MSTest tests get a `TestContext`, through their `TestContext` property or a constructor that takes one, and `[AssemblyInitialize]`, `[ClassInitialize]` and the matching cleanup methods receive one when they take it. It carries `TestName`, `FullyQualifiedTestClassName` and a `CancellationTokenSource`, and `CurrentTestOutcome` is the test's own outcome by the time `[TestCleanup]` runs. `WriteLine` goes to the console. Files passed to `AddResultFile` are left where they are rather than attached to a result.
+- `[DataRow]`, `[DynamicData]` and any other MSTest data source run as one test per row, named the way MSTest names them, such as `SignInTests.SignsIn (owner)`. `--filter` matches that name, so it can select a single row.
+- Setup methods (`[TestInitialize]`, NUnit `[SetUp]`) run base class first. Cleanup methods (`[TestCleanup]`, NUnit `[TearDown]`) run derived class first and base class last, and each one runs even when an earlier one throws.
+- Attributes derived from `[TestClass]` or `[TestMethod]`, such as `[DataTestMethod]`, `[MotusTestClass]` and `[MotusTestMethod]`, mark tests as the attributes they derive from do. NUnit `[TestCase]` and xUnit `[Theory]` tests are not run.
+
 **Exit code.** `0` when everything passed. `1` when any test failed, when a coverage threshold was missed, or when `--fail-on-flaky` was given and any test was flaky.
 
 See [Sharding](../guides/sharding.md), [Flaky Tests and Quarantine](../guides/flaky-tests-and-quarantine.md), and [Configuration](../guides/configuration.md) for the settings behind these.

src/Motus.Cli/Commands/RunCommand.cs

@@ -208,7 +208,7 @@ public static Command Build()
                 }
 
                 tests = tests
-                    .Select(t => t with { Quarantined = t.Quarantined || quarantineList.Contains(t.FullName) })
+                    .Select(t => t with { Quarantined = t.Quarantined || IsListed(t, quarantineList) })
                     .ToList();
             }
 
@@ -250,6 +250,14 @@ private static bool TryParseRetryPolicy(string spec, out RetryPolicy policy)
         }
     }
 
+    /// <summary>
+    /// True when the quarantine list names <paramref name="test"/>. A data-driven test's rows can be
+    /// listed one by one, or all at once by the method's name.
+    /// </summary>
+    internal static bool IsListed(DiscoveredTest test, IReadOnlySet<string> quarantineList)
+        => quarantineList.Contains(test.FullName)
+           || quarantineList.Contains($"{test.TestClass.FullName}.{test.TestMethod.Name}");
+
     private static HashSet<string> LoadQuarantineList(string path)
     {
         var set = new HashSet<string>(StringComparer.Ordinal);

src/Motus.Cli/Services/TestDiscovery.cs

@@ -1,8 +1,13 @@
 using System.Reflection;
+using Motus.Runner.Services.Execution;
 
 namespace Motus.Cli.Services;
 
-public sealed record DiscoveredTest(Type TestClass, MethodInfo TestMethod, string FullName, bool IsIgnored, bool Quarantined = false);
+/// <summary>
+/// A test case found in a test assembly. <paramref name="Arguments"/> is the data row of one case of a
+/// data-driven test, or null for a test without one.
+/// </summary>
+public sealed record DiscoveredTest(Type TestClass, MethodInfo TestMethod, string FullName, bool IsIgnored, bool Quarantined = false, object?[]? Arguments = null);
 
 public sealed class TestDiscovery
 {
@@ -19,6 +24,12 @@ public sealed class TestDiscovery
         "FactAttribute",
     };
 
+    // xUnit's [Theory] derives from [Fact] but needs data the runners don't supply.
+    private static readonly HashSet<string> ExcludedMethodAttributes = new(StringComparer.Ordinal)
+    {
+        "TheoryAttribute",
+    };
+
     private static readonly HashSet<string> IgnoreAttributes = new(StringComparer.Ordinal)
     {
         "IgnoreAttribute",  // MSTest + NUnit
@@ -63,14 +74,17 @@ public List<DiscoveredTest> Discover(string[] assemblyPaths, string? filter)
                     if (!IsTestMethod(method))
                         continue;
 
-                    var fullName = $"{type.FullName}.{method.Name}";
+                    var methodIgnored = classIgnored || IsIgnoredMethod(method);
+                    var quarantined = classQuarantined || IsQuarantinedMethod(method);
 
-                    if (filter is not null && !fullName.Contains(filter, StringComparison.OrdinalIgnoreCase))
-                        continue;
+                    foreach (var testCase in TestCaseExpander.Expand(type, method, message => Console.Error.WriteLine(message)))
+                    {
+                        if (filter is not null && !testCase.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase))
+                            continue;
 
-                    var isIgnored = classIgnored || IsIgnoredMethod(method);
-                    var quarantined = classQuarantined || IsQuarantinedMethod(method);
-                    tests.Add(new DiscoveredTest(type, method, fullName, isIgnored, quarantined));
+                        var isIgnored = methodIgnored || testCase.IgnoreReason is not null;
+                        tests.Add(new DiscoveredTest(type, method, testCase.FullName, isIgnored, quarantined, testCase.Arguments));
+                    }
                 }
             }
         }
@@ -84,13 +98,14 @@ private static bool IsTestClass(Type type)
             return false;
 
         return type.GetCustomAttributes(true)
-            .Any(a => ClassAttributes.Contains(a.GetType().Name));
+            .Any(a => TestCaseExpander.IsOrDerivesFrom(a, ClassAttributes));
     }
 
     private static bool IsTestMethod(MethodInfo method)
     {
         return method.GetCustomAttributes(true)
-            .Any(a => MethodAttributes.Contains(a.GetType().Name));
+            .Any(a => TestCaseExpander.IsOrDerivesFrom(a, MethodAttributes)
+                      && !TestCaseExpander.IsOrDerivesFrom(a, ExcludedMethodAttributes));
     }
 
     private static bool IsIgnoredMethod(MethodInfo method)

src/Motus.Cli/Services/TestRunner.cs

@@ -4,6 +4,7 @@
 using Motus;
 using Motus.Abstractions;
 using Motus.Cli.Services.Reporters;
+using Motus.Runner.Services.Execution;
 using CliTestResult = Motus.Cli.Services.TestResult;
 
 namespace Motus.Cli.Services;
@@ -399,93 +400,32 @@ private static bool IsTransientCdpFailure(Exception? ex)
 
     private async Task<(CliTestResult Result, Exception? Failure)> ExecuteTestAsync(DiscoveredTest test)
     {
-        var testSw = Stopwatch.StartNew();
-        object? instance = null;
-        try
-        {
-            instance = Activator.CreateInstance(test.TestClass)!;
-
-            // Publish the test method name so MotusTestBase can resolve per-method attributes
-            // (e.g. [PerformanceBudget]) and name failure artifacts without a TestContext
-            TestMethodNameContext.Set(test.TestMethod.Name);
-
-            // Run [ClassInitialize] once per type (static, takes TestContext)
-            if (_initializedClasses.TryAdd(test.TestClass, true))
+        var run = await TestCaseInvoker.RunAsync(
+            test.TestClass,
+            test.TestMethod,
+            test.Arguments,
+            DisplayNameOf(test),
+            beforeInstance: async () =>
             {
-                await RunStaticLifecycleMethodAsync(test.TestClass, "ClassInitializeAttribute");
-                await RunStaticLifecycleMethodAsync(test.TestClass, "OneTimeSetUpAttribute");
-            }
-
-            // Run [TestInitialize] / [SetUp]
-            await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TestInitializeAttribute");
-            await ExecuteLifecycleMethodAsync(instance, test.TestClass, "SetUpAttribute");
-
-            var result = test.TestMethod.Invoke(instance, null);
-
-            if (result is Task task)
-                await task;
-
-            testSw.Stop();
-            return (new CliTestResult(test.FullName, true, testSw.Elapsed, null, null), null);
-        }
-        catch (TargetInvocationException ex) when (ex.InnerException is not null)
-        {
-            testSw.Stop();
-            return (
-                new CliTestResult(test.FullName, false, testSw.Elapsed, ex.InnerException.Message, ex.InnerException.StackTrace),
-                ex.InnerException);
-        }
-        catch (Exception ex)
-        {
-            testSw.Stop();
-            return (
-                new CliTestResult(test.FullName, false, testSw.Elapsed, ex.Message, ex.StackTrace),
-                ex);
-        }
-        finally
-        {
-            if (instance is not null)
-            {
-                try
-                {
-                    // Run [TestCleanup] / [TearDown]
-                    await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TestCleanupAttribute");
-                    await ExecuteLifecycleMethodAsync(instance, test.TestClass, "TearDownAttribute");
-                }
-                catch (Exception ex)
+                // Run [ClassInitialize] once per type (static, takes TestContext)
+                if (_initializedClasses.TryAdd(test.TestClass, true))
                 {
-                    Console.Error.WriteLine($"TestCleanup failed for {test.FullName}: {ex.InnerException?.Message ?? ex.Message}");
+                    await RunStaticLifecycleMethodAsync(test.TestClass, "ClassInitializeAttribute");
+                    await RunStaticLifecycleMethodAsync(test.TestClass, "OneTimeSetUpAttribute");
                 }
+            },
+            onCleanupFailure: ex => Console.Error.WriteLine($"TestCleanup failed for {test.FullName}: {ex.Message}"));
 
-                if (instance is IAsyncDisposable asyncDisposable)
-                    await asyncDisposable.DisposeAsync();
-                else if (instance is IDisposable disposable)
-                    disposable.Dispose();
-            }
-
-            TestMethodNameContext.Clear();
-        }
+        return run.Failure is null
+            ? (new CliTestResult(test.FullName, true, run.Duration, null, null), null)
+            : (new CliTestResult(test.FullName, false, run.Duration, run.Failure.Message, run.Failure.StackTrace), run.Failure);
     }
 
-    private static async Task ExecuteLifecycleMethodAsync(object instance, Type testClass, string attributeName)
+    /// <summary>The test case's name without its class, such as <c>SignsIn ("owner")</c>.</summary>
+    private static string DisplayNameOf(DiscoveredTest test)
     {
-        // Walk the hierarchy bottom-up, then reverse so base-class methods run first
-        var methods = new List<MethodInfo>();
-        var type = testClass;
-        while (type is not null && type != typeof(object))
-        {
-            var declared = type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)
-                .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName));
-            methods.InsertRange(0, declared);
-            type = type.BaseType;
-        }
-
-        foreach (var method in methods)
-        {
-            var result = method.Invoke(instance, null);
-            if (result is Task task)
-                await task;
-        }
+        var prefix = test.TestClass.FullName + ".";
+        return test.FullName.StartsWith(prefix, StringComparison.Ordinal) ? test.FullName[prefix.Length..] : test.TestMethod.Name;
     }
 
     private static async Task RunAssemblyCleanupAsync(Assembly assembly)
@@ -496,11 +436,7 @@ private static async Task RunAssemblyCleanupAsync(Assembly assembly)
                 .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyCleanupAttribute"));
 
             foreach (var method in methods)
-            {
-                var result = method.Invoke(null, null);
-                if (result is Task task)
-                    await task;
-            }
+                await TestCaseInvoker.InvokeStaticAsync(method, type);
         }
     }
 
@@ -512,13 +448,7 @@ private static async Task RunAssemblyInitializeAsync(Assembly assembly)
                 .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == "AssemblyInitializeAttribute"));
 
             foreach (var method in methods)
-            {
-                var parameters = method.GetParameters();
-                object?[] args = parameters.Length > 0 ? [null] : [];
-                var result = method.Invoke(null, args);
-                if (result is Task task)
-                    await task;
-            }
+                await TestCaseInvoker.InvokeStaticAsync(method, type);
         }
     }
 
@@ -528,12 +458,6 @@ private static async Task RunStaticLifecycleMethodAsync(Type type, string attrib
             .Where(m => m.GetCustomAttributes(true).Any(a => a.GetType().Name == attributeName));
 
         foreach (var method in methods)
-        {
-            var parameters = method.GetParameters();
-            object?[] args = parameters.Length > 0 ? [null] : [];
-            var result = method.Invoke(null, args);
-            if (result is Task task)
-                await task;
-        }
+            await TestCaseInvoker.InvokeStaticAsync(method, type);
     }
 }

src/Motus.Runner/Components/Explorer/TestMethodNode.razor

@@ -30,7 +30,11 @@
 
     private TestStatus CurrentStatus => CurrentState?.Status ?? TestStatus.Pending;
 
-    private string MethodDisplayName => Test.TestMethod?.Name ?? GetMethodFromFullName(Test.FullName);
+    // A data row's name carries its arguments, e.g. SignsIn ("owner"), so rows of one method differ.
+    private string MethodDisplayName =>
+        Test.TestClass?.FullName is { } className && Test.FullName.StartsWith(className + ".", StringComparison.Ordinal)
+            ? Test.FullName[(className.Length + 1)..]
+            : Test.TestMethod?.Name ?? GetMethodFromFullName(Test.FullName);
 
     private bool ShowRerunButton =>
         Options.ViewerMode == ViewerMode.Trx && CurrentStatus != TestStatus.Passed;

src/Motus.Runner/Motus.Runner.csproj

@@ -13,6 +13,11 @@
     <PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="2.88.*" />
   </ItemGroup>
 
+  <ItemGroup>
+    <!-- motus run shares the visual runner's test execution code. -->
+    <InternalsVisibleTo Include="Motus.Cli" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\Motus\Motus.csproj" />
     <ProjectReference Include="..\Motus.Abstractions\Motus.Abstractions.csproj" />